How can I get a list of software in go?

linuxscoop · 2023-05-10 14:47:22

I'm interested in knowing which software packages have gained popularity recently and which ones have lost popularity. I've created a list of trending software packages in the Go programming language. Here's what I've done to get a list of trending Linux packages in the go programming language:

- Start by obtaining a list of software packages. You can get a list of all available packages from pkgstats.archlinux.de.

- Filter out any packages that are programming libraries or dependencies, as you only want to focus on actual software packages. To do this, you can use the "pacman -Qi <pkg>" command to check if the package has any dependencies. If a package does not have any dependencies, it is not a library or dependency, and you can add it to your list.

- Once you have a list of non-dependency software packages, you can determine which ones are trending. To do this, you can use the Arch Linux Package Statistics API to obtain the popularity score for each package. The popularity score is calculated based on the number of installations per month.

- Finally, sort the list of packages based on their popularity score and print the packages from the one that has gained the most popularity recently to the one that has lost the most.

The problem I have is that the list of non-dependency software is empty, and I don't know why. How can I get the list of non-dependency software in go?

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"

	"github.com/montanaflynn/stats"
)

const API string = "https://pkgstats.archlinux.de/api/"

type SomeStruct struct {
	Name   string
	Zscore float64
}

type PackagePopularity struct {
	Name       string  `json:"name"`
	Samples    int     `json:"samples"`
	Count      int     `json:"count"`
	Popularity float64 `json:"popularity"`
	StartMonth int     `json:"startMonth"`
	EndMonth   int     `json:"endMonth"`
}

type Response struct {
	Total               int                 `json:"total"`
	Count               int                 `json:"count"`
	Limit               int                 `json:"limit"`
	Offset              int                 `json:"offset"`
	Query               interface{}         `json:"query"`
	PackagePopularities []PackagePopularity `json:"packagePopularities"`
}

func main() {
	//allPackages := get_package_names()

	// Read package names from packages.txt file
	packageNames, err := readLinesFromFile("packages.txt")
	if err != nil {
		log.Fatal(err)
	}

	// Get non-dependency packages
	nonDependencyPackages := getNonDependencyPackages(packageNames)

	// Write package names to files
	//writeToFile(allPackages, "packages.txt")
  err := writeToFile(nonDependencyPackages, "non_dependency_packages.txt")
	if err != nil {
		log.Fatal(err)
	}

	// Get and print trending packages
	trending := get_trending_packages(packageNames)
	for _, _struct := range trending {
		fmt.Println(_struct.Name, _struct.Zscore)
	}
}

func get_trending_packages(pkgs []string) []SomeStruct {
	var trending []SomeStruct
	f, _ := os.Create("popularities.txt")
	defer f.Close()
	for _, pkg := range pkgs {
		popularities := get_package_popularities(pkg)
		fmt.Fprintln(f, pkg, popularities)
		trending = append(trending, SomeStruct{pkg, get_zscore(popularities)})
	}
	sort.Slice(trending, func(i, j int) bool { return trending[i].Zscore > trending[j].Zscore })
	return trending
}

func get_zscore(popularities []float64) float64 {
	mean, _ := stats.Mean(popularities)
	std, _ := stats.StandardDeviation(popularities)
	return (popularities[len(popularities)-1] - mean) / std
}

func get_total_packages() int {
	var response Response
	response = requestJSON("packages?limit=1")
	return response.Total
}

func get_package_names() []string {
	var packages []string
	var total = get_total_packages()
	var offset, limit int = 0, 100

	for offset < total {
		relURL := "packages?limit=" + fmt.Sprint(limit) + "&offset=" + fmt.Sprint(offset)
		var response Response
		response = requestJSON(relURL)
		for _, pkg := range response.PackagePopularities {
			packages = append(packages, pkg.Name)
		}
		offset += limit
	}
	return packages
}

func get_package_popularities(pkg string) []float64 {
	relURL := "packages/" + pkg + "/series?startMonth=201009"
	var response Response
	response = requestJSON(relURL)
	var popularities []float64
	for _, pkg := range response.PackagePopularities {
		popularities = append(popularities, pkg.Popularity)
	}
	return popularities
}

func requestJSON(relURL string) Response {
	resp, err := http.Get(API + relURL)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	var response Response
	err = json.Unmarshal(body, &response)
	if err != nil {
		log.Fatal(err)
	}
	return response
}

func getNonDependencyPackages(packages []string) []string {
	var nonDependencyPackages []string

	for _, pkg := range packages {
		cmd := exec.Command("pacman", "-Qi", pkg)
		output, err := cmd.Output()

		if err != nil {
			fmt.Println("Error while checking package", pkg, ":", err)
			continue
		}

		outputStr := string(output)
		if !strings.Contains(outputStr, "Required By     : None") {
			nonDependencyPackages = append(nonDependencyPackages, pkg)
		}
	}

	return nonDependencyPackages
}

func writeToFile(data []string, fileName string) error {
    file, err := os.Create(fileName)
    if err != nil {
        return err
    }
    defer file.Close()

    for _, item := range data {
        _, err = fmt.Fprintln(file, item)
        if err != nil {
            return err
        }
    }

    return nil
}

func readLinesFromFile(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lines = append(lines, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

Last edited by linuxscoop (2023-05-10 15:01:50)

Trilby · 2023-05-10 15:30:53

linuxscoop wrote:

- Filter out any packages that are programming libraries or dependencies, as you only want to focus on actual software packages. To do this, you can use the "pacman -Qi <pkg>" command to check if the package has any dependencies. If a package does not have any dependencies, it is not a library or dependency, and you can add it to your list.

There are more problems with this than I could even list. But to keep it simple, the most glaring one is the inference that if a package has no dependencies, it is not a dependency of anything else ... that's clearly nonsense.

Perhaps you meant that if a package is not a dependency of any other package, that it is not a library. This is a reasonable premise, but would also fail for your purposes as the converse is most definitely not true: there are lots of packages that are not (just) libraries that you would want on your list that are still dependencies of other packages.

Last edited by Trilby (2023-05-11 13:04:11)

linuxscoop · 2023-05-11 06:42:17

Oh right, but the question remains, is there any programatic way to get a list of actual software?

Last edited by linuxscoop (2023-05-11 06:42:41)

Lone_Wolf · 2023-05-11 08:37:43

Define what qualifies as "actual software" first.

Arch Linux

#1 2023-05-10 14:47:22

How can I get a list of software in go?

#2 2023-05-10 15:30:53

Re: How can I get a list of software in go?

#3 2023-05-11 06:42:17

Re: How can I get a list of software in go?

#4 2023-05-11 08:37:43

Re: How can I get a list of software in go?

Board footer