Merge pull request #18 from m-messiah/issue-16

Оптимизации потребления памяти

common_geobase.go

@@ -15,8 +15,7 @@ type GeoBase interface {
 	Download() ([]byte, error)
 	Unpack([]byte) error
 	Cities() (map[string]GeoItem, error)
-	Network(map[string]GeoItem) error
-	WriteMap() error
+	WriteMap(map[string]GeoItem) error
 	Name() string
 	AddError(Error)
 }
@@ -40,13 +39,7 @@ func Generate(geobase GeoBase) {
 		return
 	}
 	printMessage(geobase.Name(), "Generate cities", "OK")
-	err = geobase.Network(cities)
-	if err != nil {
-		geobase.AddError(Error{err, geobase.Name(), "Generate db"})
-		return
-	}
-	printMessage(geobase.Name(), "Generate database", "OK")
-	if err := geobase.WriteMap(); err != nil {
+	if err := geobase.WriteMap(cities); err != nil {
 		geobase.AddError(Error{err, geobase.Name(), "Write map"})
 		return
 	}

ipgeobase.go

@@ -7,13 +7,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"sort"
 )
 
 type IPGeobase struct {
 	OutputDir  string
 	ErrorsChan chan Error
-	database   map[string]GeoItem
 	archive    []*zip.File
 }
 
@@ -74,27 +72,27 @@ func (ipgeobase *IPGeobase) Cities() (map[string]GeoItem, error) {
 	return cities, nil
 }
 
-func (ipgeobase *IPGeobase) Network(cities map[string]GeoItem) error {
-	database := make(map[string]GeoItem)
-	for record := range readCSVDatabase(ipgeobase.archive, "cidr_optim.txt", "IPGeobase", '\t', true) {
-		if len(record) < 5 {
-			printMessage("IPGeobase", fmt.Sprintf("cidr_optim.txt too short line: %s", record), "FAIL")
-			continue
-		}
-		// Format is: <int_start>\t<int_end>\t<ip_range>\t<country_code>\tcity_id
-		ipRange, country, cid := removeSpace(record[2]), record[3], record[4]
-		if city, ok := cities[cid]; country == "RU" && ok {
-			database[ipRange] = city
+func (ipgeobase *IPGeobase) parseNetwork(cities map[string]GeoItem) <-chan GeoItem {
+	database := make(chan GeoItem)
+	go func() {
+		for record := range readCSVDatabase(ipgeobase.archive, "cidr_optim.txt", "IPGeobase", '\t', true) {
+			if len(record) < 5 {
+				printMessage("IPGeobase", fmt.Sprintf("cidr_optim.txt too short line: %s", record), "FAIL")
+				continue
+			}
+			// Format is: <int_start>\t<int_end>\t<ip_range>\t<country_code>\tcity_id
+			ipRange, country, cid := removeSpace(record[2]), record[3], record[4]
+			if info, ok := cities[cid]; country == "RU" && ok {
+				info.Network = ipRange
+				database <- info
+			}
 		}
-	}
-	if len(database) < 1 {
-		return errors.New("Database is empty")
-	}
-	ipgeobase.database = database
-	return nil
+		close(database)
+	}()
+	return database
 }
 
-func (ipgeobase *IPGeobase) WriteMap() error {
+func (ipgeobase *IPGeobase) WriteMap(cities map[string]GeoItem) error {
 	reg, err := openMapFile(ipgeobase.OutputDir, "region.txt")
 	if err != nil {
 		return err
@@ -110,18 +108,11 @@ func (ipgeobase *IPGeobase) WriteMap() error {
 	defer reg.Close()
 	defer city.Close()
 	defer tz.Close()
-	ipRanges := make(IPList, len(ipgeobase.database))
-	i := 0
-	for ipRange := range ipgeobase.database {
-		ipRanges[i] = ipRange
-		i++
-	}
-	sort.Sort(ipRanges)
-	for _, ipRange := range ipRanges {
-		info := ipgeobase.database[ipRange]
-		fmt.Fprintf(city, "%s %s;\n", ipRange, base64.StdEncoding.EncodeToString([]byte(info.City)))
-		fmt.Fprintf(reg, "%s %02d;\n", ipRange, info.RegID)
-		fmt.Fprintf(tz, "%s %s;\n", ipRange, info.TZ)
+
+	for info := range ipgeobase.parseNetwork(cities) {
+		fmt.Fprintf(city, "%s %s;\n", info.Network, base64.StdEncoding.EncodeToString([]byte(info.City)))
+		fmt.Fprintf(reg, "%s %02d;\n", info.Network, info.RegID)
+		fmt.Fprintf(tz, "%s %s;\n", info.Network, info.TZ)
 	}
 	return nil
 }

maxmind.go

@@ -8,7 +8,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -107,41 +106,33 @@ func (maxmind *MaxMind) Cities() (map[string]GeoItem, error) {
 	return locations, nil
 }
 
-func (maxmind *MaxMind) Network(locations map[string]GeoItem) error {
-	var database Database
-	var ipRange string
-	var geoID string
-	filename := "GeoLite2-City-Blocks-IPv" + strconv.Itoa(maxmind.ipver) + ".csv"
-	for record := range readCSVDatabase(maxmind.archive, filename, "MaxMind", ',', false) {
-		if len(record) < 2 {
-			printMessage("MaxMind", fmt.Sprintf(filename+" too short line: %s", record), "FAIL")
-			continue
-		}
-		ipRange = getIPRange(maxmind.ipver, record[0])
-		if ipRange == "" {
-			continue
-		}
-		geoID = record[1]
-		if location, ok := locations[geoID]; ok {
-			database = append(database, GeoItem{
-				ID:          geoID,
-				City:        location.City,
-				Network:     ipRange,
-				TZ:          location.TZ,
-				Country:     location.Country,
-				CountryCode: location.CountryCode,
-			})
+func (maxmind *MaxMind) parseNetwork(locations map[string]GeoItem) <-chan GeoItem {
+	database := make(chan GeoItem)
+	go func() {
+		var ipRange string
+		var geoID string
+		filename := "GeoLite2-City-Blocks-IPv" + strconv.Itoa(maxmind.ipver) + ".csv"
+		for record := range readCSVDatabase(maxmind.archive, filename, "MaxMind", ',', false) {
+			if len(record) < 2 {
+				printMessage("MaxMind", fmt.Sprintf(filename+" too short line: %s", record), "FAIL")
+				continue
+			}
+			ipRange = getIPRange(maxmind.ipver, record[0])
+			if ipRange == "" {
+				continue
+			}
+			geoID = record[1]
+			if location, ok := locations[geoID]; ok {
+				location.Network = ipRange
+				database <- location
+			}
 		}
-	}
-	if len(database) < 1 {
-		return errors.New("Network db is empty")
-	}
-	sort.Sort(database)
-	maxmind.database = database
-	return nil
+		close(database)
+	}()
+	return database
 }
 
-func (maxmind *MaxMind) WriteMap() error {
+func (maxmind *MaxMind) WriteMap(locations map[string]GeoItem) error {
 	city, err := openMapFile(maxmind.OutputDir, "mm_city.txt")
 	if err != nil {
 		return err
@@ -166,7 +157,8 @@ func (maxmind *MaxMind) WriteMap() error {
 	}
 	defer city.Close()
 	defer tz.Close()
-	for _, location := range maxmind.database {
+
+	for location := range maxmind.parseNetwork(locations) {
 		var cityName string
 		var countryName string
 		if maxmind.noBase64 {