Working artifact manager

2021-03-26 12:00:36 -04:00 · 2021-03-26 12:00:36 -04:00 · 06cb695733
parent 2ac3ee3c37
commit 06cb695733
4 changed files with 378 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,3 @@
+.idea/
+continuous/
+pkey.pem
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,8 @@
+module artifacts
+
+go 1.16
+
+require (
+	github.com/bradleyfalzon/ghinstallation v1.1.1
+	github.com/google/go-github/v33 v33.0.0
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,20 @@
+github.com/bradleyfalzon/ghinstallation v1.1.1 h1:pmBXkxgM1WeF8QYvDLT5kuQiHMcmf+X015GI0KM/E3I=
+github.com/bradleyfalzon/ghinstallation v1.1.1/go.mod h1:vyCmHTciHx/uuyN82Zc3rXN3X2KTK8nUTCrTMwAhcug=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
+github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
+github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM=
+github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
--- a/main.go
+++ b/main.go
@ -0,0 +1,347 @@
+package main
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/flate"
+	"context"
+	"fmt"
+	"github.com/bradleyfalzon/ghinstallation"
+	"github.com/google/go-github/v33/github"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const GITHUB_APP_ID = 105923
+const GITHUB_INSTALLATION_ID = 15502402
+
+const OWNER = "AxioDL"
+const REPO = "urde"
+
+func main() {
+	ex, err := os.Executable()
+	if err != nil {
+		panic(err)
+	}
+	found := false
+	pkeyPath := path.Join(filepath.Dir(ex), "pkey.pem")
+	if found, err = exists(pkeyPath); err != nil {
+		panic(err)
+	}
+	if !found {
+		pkeyPath = "pkey.pem"
+	}
+
+	itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, GITHUB_APP_ID, GITHUB_INSTALLATION_ID, pkeyPath)
+	if err != nil {
+		panic(err)
+	}
+
+	client := github.NewClient(&http.Client{Transport: itr})
+	artifacts, _, err := client.Actions.ListArtifacts(context.Background(), OWNER, REPO, &github.ListOptions{})
+	if err != nil {
+		panic(err)
+	}
+
+	var platformCompilerMap = map[string]string{
+		"linux": "gcc",
+		"macos": "appleclang",
+		"win32": "msvc",
+	}
+	var platformIndex = map[string][]string{
+		"linux": {},
+		"macos": {},
+		"win32": {},
+	}
+
+	for _, artifact := range artifacts.Artifacts {
+		split := strings.Split(*artifact.Name, "-")
+		project, version, platform, compiler, arch := split[0], split[1], split[2], split[3], split[4]
+		if project != "urde" || platformCompilerMap[platform] != compiler {
+			continue
+		}
+		fmt.Println("Selected artifact", project, version, platform, compiler, arch)
+		baseDir := fmt.Sprintf("continuous/%s", platform)
+		name := fmt.Sprintf("%s-%s-%s-%s", project, version, platform, arch)
+
+		extension := ""
+		if platform == "win32" {
+			extension = "zip"
+		} else if platform == "linux" {
+			extension = "tar"
+		} else if platform == "macos" {
+			extension = "dmg"
+		}
+
+		// Check if we've previously looked at this artifact and
+		// it didn't contain the binaries we wanted
+		skipFile := fmt.Sprintf("%s/.skip.%s.%s", baseDir, name, extension)
+		if found, err := exists(skipFile); found || err != nil {
+			if err != nil {
+				panic(err)
+			}
+			fmt.Println("Skipping bad file", name)
+			continue
+		}
+
+		// Add to platform index file
+		platformIndex[platform] = append(platformIndex[platform], fmt.Sprintf("%s.%s", name, extension))
+
+		// Check if artifact already exists in output
+		outPath := fmt.Sprintf("%s/%s.%s", baseDir, name, extension)
+		if exist, err := exists(outPath); exist || err != nil {
+			if err != nil {
+				panic(err)
+			}
+			fmt.Println("Skipping existing file", name)
+			continue
+		}
+
+		zr, err := openArtifact(client, *artifact.ID)
+		if err != nil {
+			panic(err)
+		}
+
+		found := false
+		if platform == "linux" {
+			found, err = writeLinuxTar(zr, name, baseDir)
+		} else if platform == "win32" {
+			found, err = writeWin32Zip(zr, name, baseDir)
+		} else if platform == "macos" {
+			found, err = writeMacosDmg(zr, name, baseDir)
+		}
+		if err != nil {
+			panic(err)
+		}
+
+		// If the artifact didn't contain the information we wanted,
+		// make sure we skip it in the future
+		if !found {
+			fmt.Println("Artifact skipped")
+
+			// Remove from platform index
+			platformIndex[platform] = platformIndex[platform][:len(platformIndex[platform])-1]
+
+			// Create .skip file
+			file, err := os.Create(skipFile)
+			if err != nil {
+				panic(err)
+			}
+			err = file.Close()
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+
+	for platform, names := range platformIndex {
+		file, err := createTempFile(path.Join("continuous", platform))
+		if err != nil {
+			panic(err)
+		}
+		if _, err := file.WriteString(strings.Join(names, "\n")); err != nil {
+			panic(err)
+		}
+		if err := file.Close(); err != nil {
+			panic(err)
+		}
+		if err := finalizeTempFile(file.Name(), path.Join("continuous", platform, "index.txt")); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func exists(name string) (bool, error) {
+	_, err := os.Stat(name)
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return err == nil, err
+}
+
+func openArtifact(client *github.Client, artifactID int64) (*zip.Reader, error) {
+	url, _, err := client.Actions.DownloadArtifact(context.Background(), OWNER, REPO, artifactID, true)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.Get(url.String())
+	if err != nil {
+		return nil, err
+	}
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return nil, err
+	}
+	return zip.NewReader(bytes.NewReader(body), int64(len(body)))
+}
+
+func createTempFile(dir string) (*os.File, error) {
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(dir, ".artifact")
+}
+
+func finalizeTempFile(from string, to string) error {
+	if err := os.Chmod(from, 0644); err != nil {
+		return err
+	}
+	return os.Rename(from, to)
+}
+
+func writeLinuxTar(zr *zip.Reader, name string, baseDir string) (bool, error) {
+	foundFile := false
+	for _, file := range zr.File {
+		if strings.HasSuffix(file.Name, ".AppImage") {
+			foundFile = true
+			break
+		}
+	}
+
+	if foundFile {
+		of, err := createTempFile(baseDir)
+		if err != nil {
+			return true, err
+		}
+
+		tw := tar.NewWriter(of)
+		for _, file := range zr.File {
+			if !strings.HasSuffix(file.Name, ".AppImage") {
+				fmt.Println("Unexpected file", file.Name)
+				continue
+			}
+
+			hdr := tar.Header{
+				Name:    fmt.Sprintf("%s.AppImage", name),
+				Size:    int64(file.UncompressedSize64),
+				Mode:    0777,
+				ModTime: file.Modified,
+			}
+			if err := tw.WriteHeader(&hdr); err != nil {
+				return true, err
+			}
+			if err := extractFile(file, tw); err != nil {
+				return true, err
+			}
+			break
+		}
+
+		if err := tw.Close(); err != nil {
+			return true, err
+		}
+		if err := of.Close(); err != nil {
+			return true, err
+		}
+		if err := finalizeTempFile(of.Name(), fmt.Sprintf("%s/%s.tar", baseDir, name)); err != nil {
+			return true, err
+		}
+	}
+	return foundFile, nil
+}
+
+func writeWin32Zip(zr *zip.Reader, name string, baseDir string) (bool, error) {
+	foundFile := false
+	for _, file := range zr.File {
+		if strings.HasSuffix(file.Name, ".exe") {
+			foundFile = true
+			break
+		}
+	}
+
+	if foundFile {
+		of, err := createTempFile(baseDir)
+		if err != nil {
+			return true, err
+		}
+
+		zw := zip.NewWriter(of)
+		zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
+			return flate.NewWriter(out, flate.BestCompression)
+		})
+
+		for _, file := range zr.File {
+			if !strings.HasSuffix(file.Name, ".exe") && !strings.HasSuffix(file.Name, ".pdb") {
+				fmt.Println("Unexpected file", file.Name)
+				continue
+			}
+			hdr := zip.FileHeader{
+				Name:               file.Name,
+				Modified:           file.Modified,
+				UncompressedSize64: file.UncompressedSize64,
+			}
+			w, err := zw.CreateHeader(&hdr)
+			if err != nil {
+				return true, err
+			}
+			if err := extractFile(file, w); err != nil {
+				return true, err
+			}
+		}
+
+		if err := zw.Close(); err != nil {
+			return true, err
+		}
+		if err := of.Close(); err != nil {
+			return true, err
+		}
+		if err := finalizeTempFile(of.Name(), fmt.Sprintf("%s/%s.zip", baseDir, name)); err != nil {
+			return true, err
+		}
+	}
+	return foundFile, nil
+}
+
+func writeMacosDmg(zr *zip.Reader, name string, baseDir string) (bool, error) {
+	foundFile := false
+	for _, file := range zr.File {
+		if strings.HasSuffix(file.Name, ".dmg") {
+			foundFile = true
+			break
+		}
+	}
+
+	if foundFile {
+		of, err := createTempFile(baseDir)
+		if err != nil {
+			return true, err
+		}
+
+		for _, file := range zr.File {
+			if !strings.HasSuffix(file.Name, ".dmg") {
+				fmt.Println("Unexpected file", file.Name)
+				continue
+			}
+			if err := extractFile(file, of); err != nil {
+				return true, err
+			}
+			break
+		}
+
+		if err := of.Close(); err != nil {
+			return true, err
+		}
+		if err := finalizeTempFile(of.Name(), fmt.Sprintf("%s/%s.dmg", baseDir, name)); err != nil {
+			return true, err
+		}
+	}
+	return foundFile, nil
+}
+
+func extractFile(file *zip.File, w io.Writer) error {
+	r, err := file.Open()
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	if _, err := io.Copy(w, r); err != nil {
+		return err
+	}
+	return nil
+}