Allow outdated files on mirrors

cli/commands.go

@@ -202,7 +202,7 @@ func (c *cli) CmdList(args ...string) error {
 			}
 		}
 		if *down == true {
-			if mirror.Up == true || mirror.Enabled == false {
+			if IsUp(mirror) || mirror.Enabled == false {
 				continue
 			}
 		}
@@ -234,10 +234,8 @@ func (c *cli) CmdList(args ...string) error {
 		if *state == true {
 			if mirror.Enabled == false {
 				fmt.Fprintf(w, "\tdisabled")
-			} else if mirror.Up == true {
-				fmt.Fprintf(w, "\tup")
 			} else {
-				fmt.Fprintf(w, "\tdown")
+				fmt.Fprintf(w, "\t%s", StatusString(mirror))
 			}
 			fmt.Fprintf(w, " \t(%s)", stateSince.Format(time.RFC1123))
 		}
@@ -249,6 +247,55 @@ func (c *cli) CmdList(args ...string) error {
 	return nil
 }
 
+func IsHTTPOnly(m *rpc.Mirror) bool {
+	return strings.HasPrefix(m.HttpURL, "http://")
+}
+
+func IsHTTPSOnly(m *rpc.Mirror) bool {
+	return strings.HasPrefix(m.HttpURL, "https://")
+}
+
+func IsUp(m *rpc.Mirror) bool {
+	if m.HttpUp == m.HttpsUp {
+		return m.HttpUp
+	}
+	if IsHTTPOnly(m) {
+		return m.HttpUp
+	}
+	if IsHTTPSOnly(m) {
+		return m.HttpsUp
+	}
+	return false
+}
+
+func StatusString(m *rpc.Mirror) string {
+	var http string
+	var https string
+
+	if m.HttpUp {
+		http = "up"
+	} else {
+		http = "down"
+	}
+
+	if m.HttpsUp {
+		https = "up"
+	} else {
+		https = "down"
+	}
+
+	if m.HttpUp == m.HttpsUp {
+		return http
+	}
+	if IsHTTPOnly(m) {
+		return http
+	}
+	if IsHTTPSOnly(m) {
+		return https
+	}
+	return fmt.Sprintf("%s/%s", http, https)
+}
+
 func (c *cli) CmdAdd(args ...string) error {
 	cmd := SubCmd("add", "[OPTIONS] IDENTIFIER", "Add a new mirror")
 	http := cmd.String("http", "", "HTTP base URL")
@@ -284,14 +331,25 @@ func (c *cli) CmdAdd(args ...string) error {
 		os.Exit(-1)
 	}
 
-	if !strings.HasPrefix(*http, "http://") && !strings.HasPrefix(*http, "https://") {
-		*http = "http://" + *http
-	}
-
-	_, err := url.Parse(*http)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Can't parse url\n")
+	if utils.HasAnyPrefix(*http, "http://", "https://") {
+		_, err := url.Parse(*http)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Can't parse HTTP URL\n")
+			os.Exit(-1)
+		}
+	} else if strings.Contains(*http, "://") {
+		fmt.Fprintf(os.Stderr, "The HTTP URL has an invalid scheme\n")
 		os.Exit(-1)
+	} else {
+		// No scheme, that's a "relative URLs", and we do accept it.
+		// Note that the documentation of net/url mentions that parsing
+		// such URL is "invalid but may not necessarily return an error",
+		// so let's add a scheme before we parse it.
+		_, err := url.Parse("http://" + *http)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Can't parse HTTP URL\n")
+			os.Exit(-1)
+		}
 	}
 
 	mirror := &mirrors.Mirror{
@@ -856,7 +914,12 @@ func (c *cli) CmdExport(args ...string) error {
 			urls = append(urls, m.RsyncURL)
 		}
 		if *http == true && m.HttpURL != "" {
-			urls = append(urls, m.HttpURL)
+			if utils.HasAnyPrefix(m.HttpURL, "http://", "https://") {
+				urls = append(urls, m.HttpURL)
+			} else {
+				urls = append(urls, "http://" + m.HttpURL)
+				urls = append(urls, "https://" + m.HttpURL)
+			}
 		}
 		if *ftp == true && m.FtpURL != "" {
 			urls = append(urls, m.FtpURL)
@@ -1014,10 +1077,8 @@ func (c *cli) CmdStats(args ...string) error {
 		fmt.Fprintf(w, "Identifier:\t%s\n", name)
 		if !reply.Mirror.Enabled {
 			fmt.Fprintf(w, "Status:\tdisabled\n")
-		} else if reply.Mirror.Up {
-			fmt.Fprintf(w, "Status:\tup\n")
 		} else {
-			fmt.Fprintf(w, "Status:\tdown\n")
+			fmt.Fprintf(w, "Status:\t%s\n", StatusString(reply.Mirror))
 		}
 		fmt.Fprintf(w, "Download requests:\t%d\n", reply.Requests)
 		fmt.Fprint(w, "Bytes transferred:\t")

config/config.go

@@ -37,6 +37,7 @@ func defaultConfig() Configuration {
 		OutputMode:             "auto",
 		ListenAddress:          ":8080",
 		Gzip:                   false,
+		AllowHTTPToHTTPSRedirects: true,
 		SameDownloadInterval:   600,
 		RedisAddress:           "127.0.0.1:6379",
 		RedisPassword:          "",
@@ -71,6 +72,7 @@ type Configuration struct {
 	OutputMode              string     `yaml:"OutputMode"`
 	ListenAddress           string     `yaml:"ListenAddress"`
 	Gzip                    bool       `yaml:"Gzip"`
+	AllowHTTPToHTTPSRedirects bool     `yaml:"AllowHTTPToHTTPSRedirects"`
 	SameDownloadInterval    int        `yaml:"SameDownloadInterval"`
 	RedisAddress            string     `yaml:"RedisAddress"`
 	RedisPassword           string     `yaml:"RedisPassword"`
@@ -88,6 +90,7 @@ type Configuration struct {
 	DisallowRedirects       bool       `yaml:"DisallowRedirects"`
 	WeightDistributionRange float32    `yaml:"WeightDistributionRange"`
 	DisableOnMissingFile    bool       `yaml:"DisableOnMissingFile"`
+	AllowOutdatedFiles      []OutdatedFilesConfig `yaml:"AllowOutdatedFiles"`
 	Fallbacks               []fallback `yaml:"Fallbacks"`
 
 	RedisSentinelMasterName string      `yaml:"RedisSentinelMasterName"`
@@ -113,6 +116,11 @@ type hashing struct {
 	MD5    bool `yaml:"MD5"`
 }
 
+type OutdatedFilesConfig struct {
+	Prefix  string `yaml:"Prefix"`
+	Minutes int    `yaml:"Minutes"`
+}
+
 // LoadConfig loads the configuration file if it has not yet been loaded
 func LoadConfig() {
 	if config != nil {
@@ -167,6 +175,14 @@ func ReloadConfig() error {
 	if c.RepositoryScanInterval < 0 {
 		c.RepositoryScanInterval = 0
 	}
+	for _, rule := range c.AllowOutdatedFiles {
+		if len(rule.Prefix) > 0 && rule.Prefix[0] != '/' {
+			return fmt.Errorf("AllowOutdatedFiles.Prefix must start with '/'")
+		}
+		if rule.Minutes < 0 {
+			return fmt.Errorf("AllowOutdatedFiles.Minutes must be >= 0")
+		}
+	}
 
 	if config != nil &&
 		(c.RedisAddress != config.RedisAddress ||

core/database.go

@@ -7,7 +7,7 @@ const (
 	// RedisMinimumVersion contains the minimum redis version required to run the application
 	RedisMinimumVersion = "3.2.0"
 	// DBVersion represents the current DB format version
-	DBVersion = 1
+	DBVersion = 2
 	// DBVersionKey contains the global redis key containing the DB version format
 	DBVersionKey = "MIRRORBITS_DB_VERSION"
 )

daemon/monitor.go

@@ -461,7 +461,7 @@ func (m *monitor) syncLoop() {
 				goto end
 			}
 
-			if err == nil && mir.Enabled == true && mir.Up == false {
+			if err == nil && mir.Enabled == true && mir.IsUp() == false {
 				m.healthCheckChan <- id
 			}
 
@@ -477,22 +477,43 @@ func (m *monitor) syncLoop() {
 
 // Do an actual health check against a given mirror
 func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
-	// Format log output
-	format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4)
-
 	// Get the URL to a random file available on this mirror
 	file, size, err := m.getRandomFile(mirror.ID)
 	if err != nil {
 		if err == redis.ErrNil {
 			return errMirrorNotScanned
 		} else if !database.RedisIsLoading(err) {
-			log.Warningf(format+"Error: Cannot obtain a random file: %s", mirror.Name, err)
+			log.Warningf("%s: Error: Cannot obtain a random file: %s", mirror.Name, err)
 		}
 		return err
 	}
 
+	// Perform health check(s)
+	if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") {
+		err = m.healthCheckDo(&mirror, mirror.HttpURL, file, size)
+	} else {
+		err = m.healthCheckDo(&mirror, "http://"+mirror.HttpURL, file, size)
+		err2 := m.healthCheckDo(&mirror, "https://"+mirror.HttpURL, file, size)
+		if err2 != nil {
+			err = err2
+		}
+	}
+
+	return err
+}
+
+func (m *monitor) healthCheckDo(mirror *mirrors.Mirror, url string, file string, size int64) error {
+	// Get protocol
+	proto := mirrors.HTTP
+	if strings.HasPrefix(url, "https://") {
+		proto = mirrors.HTTPS
+	}
+
+	// Format log output
+	format := "%-" + fmt.Sprintf("%d.%ds %-5s ", m.formatLongestID+4, m.formatLongestID+4, proto)
+
 	// Prepare the HTTP request
-	req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil)
+	req, err := http.NewRequest("HEAD", strings.TrimRight(url, "/")+file, nil)
 	req.Header.Set("User-Agent", userAgent)
 	req.Close = true
 
@@ -506,7 +527,7 @@ func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
 	go func() {
 		select {
 		case <-m.stop:
-			log.Debugf("Aborting health-check for %s", mirror.HttpURL)
+			log.Debugf("Aborting health-check for %s", url)
 			cancel()
 		case <-ctx.Done():
 		}
@@ -536,7 +557,7 @@ func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
 		if strings.Contains(err.Error(), errRedirect.Error()) {
 			reason = "Unauthorized redirect"
 		}
-		markErr := mirrors.MarkMirrorDown(m.redis, mirror.ID, reason)
+		markErr := mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, reason)
 		if markErr != nil {
 			log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, markErr)
 		}
@@ -546,7 +567,7 @@ func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
 
 	switch statusCode {
 	case 200:
-		err = mirrors.MarkMirrorUp(m.redis, mirror.ID)
+		err = mirrors.MarkMirrorUp(m.redis, mirror.ID, proto)
 		if err != nil {
 			log.Errorf(format+"Unable to mark mirror as up: %s", mirror.Name, err)
 		}
@@ -557,7 +578,7 @@ func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
 			log.Noticef(format+"Up! (%dms)", mirror.Name, elapsed/time.Millisecond)
 		}
 	case 404:
-		err = mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file))
+		err = mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, fmt.Sprintf("File not found %s (error 404)", file))
 		if err != nil {
 			log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, err)
 		}
@@ -569,7 +590,7 @@ func (m *monitor) healthCheck(mirror mirrors.Mirror) error {
 		}
 		log.Errorf(format+"Error: File %s not found (error 404)", mirror.Name, file)
 	default:
-		err = mirrors.MarkMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", statusCode))
+		err = mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, fmt.Sprintf("Got status code %d", statusCode))
 		if err != nil {
 			log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, err)
 		}

database/upgrader/upgrader.go

@@ -6,6 +6,7 @@ package upgrader
 import (
 	"github.com/etix/mirrorbits/database/interfaces"
 	v1 "github.com/etix/mirrorbits/database/v1"
+	v2 "github.com/etix/mirrorbits/database/v2"
 )
 
 // Upgrader is an interface to implement a database upgrade strategy
@@ -18,6 +19,8 @@ func GetUpgrader(redis interfaces.Redis, version int) Upgrader {
 	switch version {
 	case 1:
 		return v1.NewUpgraderV1(redis)
+	case 2:
+		return v2.NewUpgraderV2(redis)
 	}
 	return nil
 }