UPDATE: support for reading db from a file

Author: fileformat

.gitignore

@@ -10,5 +10,6 @@ _site/
 .sass-cache/
 tmp/
 *.tmp
+top-1m.csv.zip
 vendor/
 .wrangler/

Dockerfile

@@ -2,8 +2,10 @@
 FROM golang:1.22-alpine as builder
 RUN apk update && \
     apk upgrade && \
-    apk --no-cache add git upx ca-certificates
+    apk --no-cache add git upx wget ca-certificates
 RUN mkdir /build
+RUN wget https://tranco-list.eu/top-1m.csv.zip
+
 ADD . /build/
 WORKDIR /build
 ARG COMMIT
@@ -17,9 +19,9 @@ RUN \
     -o siterank \
     *.go \
     && upx siterank
-
 FROM scratch
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /build/top-1m.csv.zip /app/
 COPY --from=builder /build/siterank /app/
 ENV PORT 4000
 ENTRYPOINT ["/app/siterank"]

rankings.go

@@ -4,42 +4,93 @@ import (
 	"archive/zip"
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var rankings map[string]int = make(map[string]int)
+var rankDate string
+
+func getLastModified(resp *http.Response) time.Time {
+	lastModified, lmErr := http.ParseTime(resp.Header.Get("Last-Modified"))
+	if lmErr == nil {
+		return lastModified
+	}
+	dateFromHeader, dateErr := time.Parse("2006-01-02 15:04:05", resp.Header.Get("Date"))
+	if dateErr == nil {
+		return dateFromHeader
+	}
+
+	return time.Now()
+}
 
 func initRankings() error {
 
-	url := os.Getenv("URL")
-	if url == "" {
-		url = "https://tranco-list.eu/top-1m.csv.zip"
-		//"https://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
+	urlStr := os.Getenv("URL")
+	if urlStr == "" {
+		urlStr = "file:top-1m.csv.zip"
 	}
 
-	logger.Info("Downloading rankings", "url", url)
-	resp, err := http.Get(url)
-	if err != nil {
-		logger.Error("Failed to download rankings", "error", err, "url", url)
-		return err
+	logger.Info("Loading rankings", "url", urlStr)
+	url, urlErr := url.Parse(urlStr)
+	if urlErr != nil {
+		logger.Error("Invalid URL", "error", urlErr, "url", urlStr)
+		return urlErr
 	}
-	defer resp.Body.Close()
+	logger.Debug("Parsed rankings URL", "url", urlStr, "parsed", url)
 
-	body, err := io.ReadAll(resp.Body)
+	var body []byte
+	var readErr error
+	if url.Scheme == "file" {
+		var filePath string
+		if url.Path == "" {
+			filePath = url.Opaque
+		} else {
+			filePath = url.Path
+		}
+		stats, statErr := os.Stat(filePath)
+		if statErr != nil {
+			logger.Error("Failed to stat rankings file", "error", statErr, "url", urlStr, "path", filePath)
+			return statErr
+		}
+		rankDate = stats.ModTime().Format("2006-01-02 15:04:05")
+		body, readErr = os.ReadFile(filePath)
+	} else if url.Scheme == "http" || url.Scheme == "https" {
+		resp, err := http.Get(urlStr)
+		if err != nil {
+			logger.Error("Failed to download rankings", "error", err, "url", urlStr)
+			return err
+		}
+		defer resp.Body.Close()
+
+		rankDate = getLastModified(resp).Format("2006-01-02 15:04:05")
+		body, readErr = io.ReadAll(resp.Body)
+
+		logger.Info("Downloaded rankings", "url", urlStr, "size", len(body), "contentLength", resp.ContentLength)
+	} else {
+		logger.Error("Unsupported URL scheme", "scheme", url.Scheme, "url", urlStr)
+		return fmt.Errorf("unsupported URL scheme %s", url.Scheme)
+	}
+	if readErr != nil {
+		logger.Error("Failed to read rankings", "error", readErr, "url", urlStr)
+		return readErr
+	}
 
-	zipData, err := zip.NewReader(bytes.NewReader(body), resp.ContentLength)
+	zipData, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	if err != nil {
-		logger.Error("Failed to read zip file", "error", err, "url", url)
+		logger.Error("Failed to read zip file", "error", err, "url", urlStr)
 		return err
 	}
 
 	zipFile, zipFileErr := zipData.File[0].Open()
 	if zipFileErr != nil {
-		logger.Error("Failed to open file from zip", "error", zipFileErr, "url", url)
+		logger.Error("Failed to open file from zip", "error", zipFileErr, "url", urlStr)
 		return zipFileErr
 	}
 
@@ -50,21 +101,21 @@ func initRankings() error {
 		line := scanner.Text()
 		parts := strings.Split(line, ",")
 		if len(parts) != 2 {
-			logger.Error("Invalid ranking line", "line", line, "lineNumber", lineNo, "url", url)
+			logger.Error("Invalid ranking line", "line", line, "lineNumber", lineNo, "url", urlStr)
 			continue
 		}
 		rank, err := strconv.Atoi(parts[0])
 		if err != nil {
-			logger.Error("Invalid ranking rank", "line", line, "lineNumber", lineNo, "url", url)
+			logger.Error("Invalid ranking rank", "line", line, "lineNumber", lineNo, "url", urlStr)
 			continue
 		}
 		rankings[parts[1]] = rank
 	}
 	if err := scanner.Err(); err != nil {
-		logger.Error("Failed to scan zip file", "error", err, "url", url)
+		logger.Error("Failed to scan zip file", "error", err, "url", urlStr)
 		return err
 	}
-	logger.Info("Loaded rankings", "url", url, "count", len(rankings))
+	logger.Info("Loaded rankings", "url", urlStr, "count", len(rankings))
 
 	return nil
 }

static/index.html

@@ -15,6 +15,9 @@ <h1>
         <p>
             API to retrieve the rank of a website in a given ranking list.
         </p>
+        <p>
+            Try it via the form at <a href="https://resolve.rs/domains/rank.html">resolve.rs</a>!
+        </p>
         <p>
             <a href="https://github.com/redirect2me/siterank">How this works</a>, including API details and source code!
         </p>

statusHandler.go

@@ -16,6 +16,7 @@ type Status struct {
 	Commit    string `json:"commit"`
 	LastMod   string `json:"lastmod"`
 	Tech      string `json:"tech"`
+	RankDate  string `json:"rankings"`
 }
 
 func statusHandler(w http.ResponseWriter, r *http.Request) {
@@ -27,6 +28,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request) {
 	status.Commit = COMMIT
 	status.LastMod = LASTMOD
 	status.Tech = runtime.Version()
+	status.RankDate = rankDate
 
 	handleJson(w, r, status)
 }