opera-proxy/freeproxy_fetch.go

package main

import (
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"
)

var (
	freeProxyIPPattern   = regexp.MustCompile(`data-ip="([^"]+)"`)
	freeProxyPortPattern = regexp.MustCompile(`data-port="([^"]+)"`)
)

func fetchFreeProxyToFile(sourceURL, outPath string, timeout time.Duration) (int, error) {
	proxies, err := fetchFreeProxyList(sourceURL, timeout)
	if err != nil {
		return 0, err
	}
	if len(proxies) == 0 {
		return 0, fmt.Errorf("no proxies found at %s", sourceURL)
	}
	if err := os.WriteFile(outPath, []byte(strings.Join(proxies, "\n")+"\n"), 0o644); err != nil {
		return 0, err
	}
	return len(proxies), nil
}

func fetchFreeProxyList(sourceURL string, timeout time.Duration) ([]string, error) {
	client := &http.Client{Timeout: timeout}
	seen := make(map[string]struct{})
	proxies := make([]string, 0, 256)

	for page := 1; ; page++ {
		pageURL, err := freeProxyPageURL(sourceURL, page)
		if err != nil {
			return nil, err
		}

		pageProxies, err := fetchFreeProxyPage(client, pageURL)
		if err != nil {
			return nil, fmt.Errorf("page %d: %w", page, err)
		}
		if len(pageProxies) == 0 {
			break
		}

		for _, proxy := range pageProxies {
			if _, ok := seen[proxy]; ok {
				continue
			}
			seen[proxy] = struct{}{}
			proxies = append(proxies, proxy)
		}
	}

	return proxies, nil
}

func freeProxyPageURL(rawURL string, page int) (string, error) {
	parsed, err := url.Parse(rawURL)
	if err != nil {
		return "", fmt.Errorf("parse url: %w", err)
	}

	query := parsed.Query()
	if page <= 1 {
		query.Del("page")
	} else {
		query.Set("page", fmt.Sprintf("%d", page))
	}
	parsed.RawQuery = query.Encode()

	return parsed.String(), nil
}

func fetchFreeProxyPage(client *http.Client, pageURL string) ([]string, error) {
	req, err := http.NewRequest(http.MethodGet, pageURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "opera-proxy freeproxy fetcher/1.0")

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpected status %s", resp.Status)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	ipMatches := freeProxyIPPattern.FindAllSubmatch(body, -1)
	portMatches := freeProxyPortPattern.FindAllSubmatch(body, -1)
	rowCount := min(len(ipMatches), len(portMatches))

	proxies := make([]string, 0, rowCount)
	for i := 0; i < rowCount; i++ {
		ip, err := freeProxyDecodeBase64(string(ipMatches[i][1]))
		if err != nil {
			return nil, fmt.Errorf("decode ip on row %d: %w", i+1, err)
		}
		port, err := freeProxyDecodeBase64(string(portMatches[i][1]))
		if err != nil {
			return nil, fmt.Errorf("decode port on row %d: %w", i+1, err)
		}
		proxies = append(proxies, ip+":"+port)
	}

	return proxies, nil
}

func freeProxyDecodeBase64(value string) (string, error) {
	decoded, err := base64.StdEncoding.DecodeString(value)
	if err != nil {
		return "", err
	}
	return string(decoded), nil
}