Add new API settings

freeproxy_fetch.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var (
+	freeProxyIPPattern   = regexp.MustCompile(`data-ip="([^"]+)"`)
+	freeProxyPortPattern = regexp.MustCompile(`data-port="([^"]+)"`)
+)
+
+func fetchFreeProxyToFile(sourceURL, outPath string, timeout time.Duration) (int, error) {
+	proxies, err := fetchFreeProxyList(sourceURL, timeout)
+	if err != nil {
+		return 0, err
+	}
+	if len(proxies) == 0 {
+		return 0, fmt.Errorf("no proxies found at %s", sourceURL)
+	}
+	if err := os.WriteFile(outPath, []byte(strings.Join(proxies, "\n")+"\n"), 0o644); err != nil {
+		return 0, err
+	}
+	return len(proxies), nil
+}
+
+func fetchFreeProxyList(sourceURL string, timeout time.Duration) ([]string, error) {
+	client := &http.Client{Timeout: timeout}
+	seen := make(map[string]struct{})
+	proxies := make([]string, 0, 256)
+
+	for page := 1; ; page++ {
+		pageURL, err := freeProxyPageURL(sourceURL, page)
+		if err != nil {
+			return nil, err
+		}
+
+		pageProxies, err := fetchFreeProxyPage(client, pageURL)
+		if err != nil {
+			return nil, fmt.Errorf("page %d: %w", page, err)
+		}
+		if len(pageProxies) == 0 {
+			break
+		}
+
+		for _, proxy := range pageProxies {
+			if _, ok := seen[proxy]; ok {
+				continue
+			}
+			seen[proxy] = struct{}{}
+			proxies = append(proxies, proxy)
+		}
+	}
+
+	return proxies, nil
+}
+
+func freeProxyPageURL(rawURL string, page int) (string, error) {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("parse url: %w", err)
+	}
+
+	query := parsed.Query()
+	if page <= 1 {
+		query.Del("page")
+	} else {
+		query.Set("page", fmt.Sprintf("%d", page))
+	}
+	parsed.RawQuery = query.Encode()
+
+	return parsed.String(), nil
+}
+
+func fetchFreeProxyPage(client *http.Client, pageURL string) ([]string, error) {
+	req, err := http.NewRequest(http.MethodGet, pageURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "opera-proxy freeproxy fetcher/1.0")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	ipMatches := freeProxyIPPattern.FindAllSubmatch(body, -1)
+	portMatches := freeProxyPortPattern.FindAllSubmatch(body, -1)
+	rowCount := min(len(ipMatches), len(portMatches))
+
+	proxies := make([]string, 0, rowCount)
+	for i := 0; i < rowCount; i++ {
+		ip, err := freeProxyDecodeBase64(string(ipMatches[i][1]))
+		if err != nil {
+			return nil, fmt.Errorf("decode ip on row %d: %w", i+1, err)
+		}
+		port, err := freeProxyDecodeBase64(string(portMatches[i][1]))
+		if err != nil {
+			return nil, fmt.Errorf("decode port on row %d: %w", i+1, err)
+		}
+		proxies = append(proxies, ip+":"+port)
+	}
+
+	return proxies, nil
+}
+
+func freeProxyDecodeBase64(value string) (string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(value)
+	if err != nil {
+		return "", err
+	}
+	return string(decoded), nil
+}