diff --git a/10proxies.txt b/10proxies.txt new file mode 100644 index 0000000..af9564b --- /dev/null +++ b/10proxies.txt @@ -0,0 +1,10 @@ +31.59.20.176:6754:oxcestpe:2yoo1hy06sho +198.23.239.134:6540:oxcestpe:2yoo1hy06sho +45.38.107.97:6014:oxcestpe:2yoo1hy06sho +107.172.163.27:6543:oxcestpe:2yoo1hy06sho +198.105.121.200:6462:oxcestpe:2yoo1hy06sho +216.10.27.159:6837:oxcestpe:2yoo1hy06sho +142.111.67.146:5611:oxcestpe:2yoo1hy06sho +191.96.254.138:6185:oxcestpe:2yoo1hy06sho +31.58.9.4:6077:oxcestpe:2yoo1hy06sho +23.26.71.145:5628:oxcestpe:2yoo1hy06sho diff --git a/README.md b/README.md index 61ba091..ef5fcc8 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,44 @@ eu2.sec-tunnel.com,77.111.247.51,443 eu3.sec-tunnel.com,77.111.244.22,443 ``` +You can also skip the SurfEasy discover request and take endpoints from an existing CSV, while still using the normal server selection logic including `-server-selection fastest`: + +``` +$ ./opera-proxy -discover-csv proxies.csv -server-selection fastest +``` + +If direct SurfEasy API access is unstable, you can point discovery at a text file with fallback proxies. The app will read `proxies.txt`, test API proxies in parallel, and stop on the first one that successfully completes init/discover. File entries may be plain `host:port`, URL-style `http://user:pass@host:port`, or `host:port:user:pass` like `10proxies.txt`: + +``` +$ ./opera-proxy -api-proxy-file proxies.txt -country EU +``` + +By default it tests up to 5 candidates at once. You can change that with `-api-proxy-parallel`: + +``` +$ ./opera-proxy -api-proxy-file proxies.txt -api-proxy-parallel 5 -country EU +``` + +You can also download the proxy list from a URL. If the download fails, the app can fall back to a local file: + +``` +$ ./opera-proxy -api-proxy-list-url https://example.com/proxies.txt -country EU +``` + +``` +$ ./opera-proxy -api-proxy-list-url https://example.com/proxies.txt -api-proxy-file proxies.txt -country EU +``` + +If you want selected destinations to go directly instead of through the Opera proxy, use `-proxy-bypass`. It accepts a comma-separated list of host or URL patterns and supports `*` in hostnames: + +``` +$ ./opera-proxy -country EU -proxy-bypass "api2.sec-tunnel.com,*.example.com,https://download.test.local/list.txt" +``` + +If `-proxy-bypass` is not passed, the app also tries to read `proxy-bypass.txt` from the current working directory. The file supports one pattern per line, comments via `#`, and comma-separated values on the same line. + +If SurfEasy discover returns API error `801`, the app also automatically tries `proxies.csv` from the current working directory, even when `-discover-csv` was not passed. + ## List of arguments | Argument | Type | Description | @@ -70,19 +108,25 @@ eu3.sec-tunnel.com,77.111.244.22,443 | api-login | String | SurfEasy API login (default "se0316") | | api-password | String | SurfEasy API password (default "SILrMEPBmJuhomxWkfm3JalqHX2Eheg1YhlEZiMh8II") | | api-proxy | String | additional proxy server used to access SurfEasy API | +| api-proxy-file | String | path to text file with candidate proxy servers for SurfEasy API access, one per line; proxies are tried in order until init/discover succeeds | +| api-proxy-list-url | String | URL of a text file with candidate proxy servers for SurfEasy API access; falls back to `-api-proxy-file` if download fails | +| api-proxy-parallel | Number | number of API proxy candidates tested in parallel when `-api-proxy-file` is used (default 5) | | api-user-agent | String | user agent reported to SurfEasy API (default "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 OPR/114.0.0.0") | | bind-address | String | proxy listen address (default "127.0.0.1:18080") | | bootstrap-dns | String | Comma-separated list of DNS/DoH/DoT resolvers for initial discovery of SurfEasy API address. Supported schemes are: `dns://`, `https://`, `tls://`, `tcp://`. Examples: `https://1.1.1.1/dns-query`, `tls://9.9.9.9:853` (default `https://1.1.1.3/dns-query,https://8.8.8.8/dns-query,https://dns.google/dns-query,https://security.cloudflare-dns.com/dns-query,https://fidelity.vm-0.com/q,https://wikimedia-dns.org/dns-query,https://dns.adguard-dns.com/dns-query,https://dns.quad9.net/dns-query,https://doh.cleanbrowsing.org/doh/adult-filter/`) | | cafile | String | use custom CA certificate bundle file | | config | String | read configuration from file with space-separated keys and values | | country | String | desired proxy location (default "EU") | +| discover-csv | String | read proxy endpoints from CSV instead of SurfEasy discover API | | dp-export | - | export configuration for dumbproxy | -| fake-SNI | String | domain name to use as SNI in communications with servers | +| fake-SNI | String | domain name to use as SNI in outbound TLS and in tunneled TLS ClientHello when possible | | init-retries | Number | number of attempts for initialization steps, zero for unlimited retry | | init-retry-interval | Duration | delay between initialization retries (default 5s) | | list-countries | - | list available countries and exit | | list-proxies | - | output proxy list and exit | | override-proxy-address | string | use fixed proxy address instead of server address returned by SurfEasy API | +| proxy-bypass | String | comma-separated list of destination host or URL patterns that should bypass proxying and connect directly; matching is case-insensitive and supports `*` in hostnames; if omitted, `proxy-bypass.txt` from the current working directory is loaded automatically when present | +| proxy-blacklist | String | path to file with blacklisted proxy addresses, one `host[:port]` per line | | proxy | String | sets base proxy to use for all dial-outs. Format: `://[login:password@]host[:port]` Examples: `http://user:password@192.168.1.1:3128`, `socks5://10.0.0.1:1080` | | refresh | Duration | login refresh interval (default 4h0m0s) | | refresh-retry | Duration | login refresh retry interval (default 5s) | diff --git a/dialer/bypass.go b/dialer/bypass.go new file mode 100644 index 0000000..521748c --- /dev/null +++ b/dialer/bypass.go @@ -0,0 +1,133 @@ +package dialer + +import ( + "context" + "fmt" + "net" + "net/url" + "path" + "strings" +) + +type bypassPattern struct { + raw string + host string +} + +type BypassDialer struct { + direct ContextDialer + proxied ContextDialer + patterns []bypassPattern +} + +func NewBypassDialer(patterns []string, direct, proxied ContextDialer) (*BypassDialer, error) { + compiled := make([]bypassPattern, 0, len(patterns)) + for _, raw := range patterns { + hostPattern, err := normalizeBypassPattern(raw) + if err != nil { + return nil, fmt.Errorf("invalid proxy bypass pattern %q: %w", raw, err) + } + if _, err := path.Match(hostPattern, ""); err != nil { + return nil, fmt.Errorf("invalid proxy bypass pattern %q: %w", raw, err) + } + compiled = append(compiled, bypassPattern{ + raw: raw, + host: hostPattern, + }) + } + return &BypassDialer{ + direct: direct, + proxied: proxied, + patterns: compiled, + }, nil +} + +func (d *BypassDialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *BypassDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if d.shouldBypass(address) { + return d.direct.DialContext(ctx, network, address) + } + return d.proxied.DialContext(ctx, network, address) +} + +func (d *BypassDialer) shouldBypass(address string) bool { + host := normalizeDialTarget(address) + if host == "" { + return false + } + for _, pattern := range d.patterns { + matched, err := path.Match(pattern.host, host) + if err != nil { + continue + } + if matched { + return true + } + } + return false +} + +func normalizeBypassPattern(raw string) (string, error) { + pattern := strings.TrimSpace(raw) + if pattern == "" { + return "", fmt.Errorf("empty pattern") + } + + switch { + case strings.Contains(pattern, "://"): + parsed, err := url.Parse(pattern) + if err != nil { + return "", fmt.Errorf("unable to parse URL: %w", err) + } + if parsed.Host == "" { + return "", fmt.Errorf("URL does not contain a host") + } + pattern = parsed.Hostname() + case strings.HasPrefix(pattern, "//"): + parsed, err := url.Parse("https:" + pattern) + if err != nil { + return "", fmt.Errorf("unable to parse URL: %w", err) + } + if parsed.Host == "" { + return "", fmt.Errorf("URL does not contain a host") + } + pattern = parsed.Hostname() + case strings.ContainsAny(pattern, "/?"): + parsed, err := url.Parse("https://" + strings.TrimLeft(pattern, "/")) + if err != nil { + return "", fmt.Errorf("unable to parse URL-like pattern: %w", err) + } + if parsed.Host == "" { + return "", fmt.Errorf("pattern does not contain a host") + } + pattern = parsed.Hostname() + default: + if host, _, err := net.SplitHostPort(pattern); err == nil { + pattern = host + } + } + + pattern = strings.Trim(pattern, "[]") + pattern = strings.ToLower(pattern) + if pattern == "" { + return "", fmt.Errorf("empty host pattern") + } + return pattern, nil +} + +func normalizeDialTarget(address string) string { + host := strings.TrimSpace(address) + if host == "" { + return "" + } + if parsed, err := url.Parse(host); err == nil && parsed.Host != "" { + host = parsed.Hostname() + } else if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.Trim(host, "[]") + return strings.ToLower(host) +} diff --git a/dialer/bypass_test.go b/dialer/bypass_test.go new file mode 100644 index 0000000..d1c0319 --- /dev/null +++ b/dialer/bypass_test.go @@ -0,0 +1,67 @@ +package dialer + +import ( + "context" + "errors" + "net" + "testing" +) + +type recordingDialer struct { + name string + addresses []string +} + +func (d *recordingDialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *recordingDialer) DialContext(_ context.Context, _ string, address string) (net.Conn, error) { + d.addresses = append(d.addresses, address) + return nil, errors.New(d.name) +} + +func TestBypassDialerRoutesByHostPattern(t *testing.T) { + direct := &recordingDialer{name: "direct"} + proxied := &recordingDialer{name: "proxied"} + + d, err := NewBypassDialer([]string{ + "api2.sec-tunnel.com", + "*.example.com", + "https://already.url.test/some/path", + }, direct, proxied) + if err != nil { + t.Fatalf("NewBypassDialer() error = %v", err) + } + + tests := []struct { + address string + wantDirect bool + description string + }{ + {address: "api2.sec-tunnel.com:443", wantDirect: true, description: "exact host"}, + {address: "cdn.example.com:8443", wantDirect: true, description: "wildcard host"}, + {address: "ALREADY.URL.TEST:443", wantDirect: true, description: "case insensitive"}, + {address: "other.test:443", wantDirect: false, description: "unmatched host"}, + } + + for _, tt := range tests { + _, err := d.DialContext(context.Background(), "tcp", tt.address) + if err == nil { + t.Fatalf("%s: expected dial error", tt.description) + } + if tt.wantDirect && err.Error() != "direct" { + t.Fatalf("%s: expected direct dialer, got %v", tt.description, err) + } + if !tt.wantDirect && err.Error() != "proxied" { + t.Fatalf("%s: expected proxied dialer, got %v", tt.description, err) + } + } +} + +func TestNewBypassDialerRejectsInvalidPattern(t *testing.T) { + _, err := NewBypassDialer([]string{" "}, &recordingDialer{}, &recordingDialer{}) + if err == nil { + t.Fatal("expected invalid pattern error") + } +} diff --git a/freeproxy_fetch.go b/freeproxy_fetch.go new file mode 100644 index 0000000..eae67f9 --- /dev/null +++ b/freeproxy_fetch.go @@ -0,0 +1,130 @@ +package main + +import ( + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" +) + +var ( + freeProxyIPPattern = regexp.MustCompile(`data-ip="([^"]+)"`) + freeProxyPortPattern = regexp.MustCompile(`data-port="([^"]+)"`) +) + +func fetchFreeProxyToFile(sourceURL, outPath string, timeout time.Duration) (int, error) { + proxies, err := fetchFreeProxyList(sourceURL, timeout) + if err != nil { + return 0, err + } + if len(proxies) == 0 { + return 0, fmt.Errorf("no proxies found at %s", sourceURL) + } + if err := os.WriteFile(outPath, []byte(strings.Join(proxies, "\n")+"\n"), 0o644); err != nil { + return 0, err + } + return len(proxies), nil +} + +func fetchFreeProxyList(sourceURL string, timeout time.Duration) ([]string, error) { + client := &http.Client{Timeout: timeout} + seen := make(map[string]struct{}) + proxies := make([]string, 0, 256) + + for page := 1; ; page++ { + pageURL, err := freeProxyPageURL(sourceURL, page) + if err != nil { + return nil, err + } + + pageProxies, err := fetchFreeProxyPage(client, pageURL) + if err != nil { + return nil, fmt.Errorf("page %d: %w", page, err) + } + if len(pageProxies) == 0 { + break + } + + for _, proxy := range pageProxies { + if _, ok := seen[proxy]; ok { + continue + } + seen[proxy] = struct{}{} + proxies = append(proxies, proxy) + } + } + + return proxies, nil +} + +func freeProxyPageURL(rawURL string, page int) (string, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("parse url: %w", err) + } + + query := parsed.Query() + if page <= 1 { + query.Del("page") + } else { + query.Set("page", fmt.Sprintf("%d", page)) + } + parsed.RawQuery = query.Encode() + + return parsed.String(), nil +} + +func fetchFreeProxyPage(client *http.Client, pageURL string) ([]string, error) { + req, err := http.NewRequest(http.MethodGet, pageURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "opera-proxy freeproxy fetcher/1.0") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + ipMatches := freeProxyIPPattern.FindAllSubmatch(body, -1) + portMatches := freeProxyPortPattern.FindAllSubmatch(body, -1) + rowCount := min(len(ipMatches), len(portMatches)) + + proxies := make([]string, 0, rowCount) + for i := 0; i < rowCount; i++ { + ip, err := freeProxyDecodeBase64(string(ipMatches[i][1])) + if err != nil { + return nil, fmt.Errorf("decode ip on row %d: %w", i+1, err) + } + port, err := freeProxyDecodeBase64(string(portMatches[i][1])) + if err != nil { + return nil, fmt.Errorf("decode port on row %d: %w", i+1, err) + } + proxies = append(proxies, ip+":"+port) + } + + return proxies, nil +} + +func freeProxyDecodeBase64(value string) (string, error) { + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return "", err + } + return string(decoded), nil +} diff --git a/handler/bypass_test.go b/handler/bypass_test.go new file mode 100644 index 0000000..45d2099 --- /dev/null +++ b/handler/bypass_test.go @@ -0,0 +1,93 @@ +package handler + +import ( + "context" + "errors" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/Snawoot/opera-proxy/dialer" + clog "github.com/Snawoot/opera-proxy/log" +) + +type recordingDialer struct { + name string + addresses []string +} + +func (d *recordingDialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *recordingDialer) DialContext(_ context.Context, _ string, address string) (net.Conn, error) { + d.addresses = append(d.addresses, address) + return nil, errors.New(d.name) +} + +func testProxyLogger() *clog.CondLogger { + return clog.NewCondLogger(log.New(io.Discard, "", 0), clog.CRITICAL) +} + +func TestProxyHandlerBypassesHTTPRequestsByTargetHost(t *testing.T) { + direct := &recordingDialer{name: "direct"} + proxied := &recordingDialer{name: "proxied"} + bypassDialer, err := dialer.NewBypassDialer([]string{"*.example.com"}, direct, proxied) + if err != nil { + t.Fatalf("NewBypassDialer() error = %v", err) + } + + h := NewProxyHandler(bypassDialer, testProxyLogger(), "") + req := httptest.NewRequest(http.MethodGet, "http://check.example.com/path", nil) + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusInternalServerError { + t.Fatalf("ServeHTTP() status = %d, want %d", rr.Code, http.StatusInternalServerError) + } + if len(direct.addresses) != 1 || direct.addresses[0] != "check.example.com:80" { + t.Fatalf("direct dialer addresses = %#v, want []string{\"check.example.com:80\"}", direct.addresses) + } + if len(proxied.addresses) != 0 { + t.Fatalf("proxied dialer should not be used, got %#v", proxied.addresses) + } +} + +func TestProxyHandlerBypassesConnectRequestsByTargetHost(t *testing.T) { + direct := &recordingDialer{name: "direct"} + proxied := &recordingDialer{name: "proxied"} + bypassDialer, err := dialer.NewBypassDialer([]string{"*.example.com"}, direct, proxied) + if err != nil { + t.Fatalf("NewBypassDialer() error = %v", err) + } + + h := NewProxyHandler(bypassDialer, testProxyLogger(), "") + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Host: "secure.example.com:443"}, + Host: "secure.example.com:443", + RequestURI: "secure.example.com:443", + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + } + rr := httptest.NewRecorder() + + h.ServeHTTP(rr, req) + + if rr.Code != http.StatusBadGateway { + t.Fatalf("ServeHTTP() status = %d, want %d", rr.Code, http.StatusBadGateway) + } + if len(direct.addresses) != 1 || direct.addresses[0] != "secure.example.com:443" { + t.Fatalf("direct dialer addresses = %#v, want []string{\"secure.example.com:443\"}", direct.addresses) + } + if len(proxied.addresses) != 0 { + t.Fatalf("proxied dialer should not be used, got %#v", proxied.addresses) + } +} diff --git a/handler/handler.go b/handler/handler.go index 09c4793..420d61f 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -25,9 +25,10 @@ type ProxyHandler struct { logger *clog.CondLogger dialer dialer.ContextDialer httptransport http.RoundTripper + fakeSNI string } -func NewProxyHandler(dialer dialer.ContextDialer, logger *clog.CondLogger) *ProxyHandler { +func NewProxyHandler(dialer dialer.ContextDialer, logger *clog.CondLogger, fakeSNI string) *ProxyHandler { httptransport := &http.Transport{ MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, @@ -39,6 +40,7 @@ func NewProxyHandler(dialer dialer.ContextDialer, logger *clog.CondLogger) *Prox logger: logger, dialer: dialer, httptransport: httptransport, + fakeSNI: fakeSNI, } } @@ -53,7 +55,7 @@ func (s *ProxyHandler) HandleTunnel(wr http.ResponseWriter, req *http.Request) { if req.ProtoMajor == 0 || req.ProtoMajor == 1 { // Upgrade client connection - localconn, _, err := hijack(wr) + localconn, rw, err := hijack(wr) if err != nil { s.logger.Error("Can't hijack client connection: %v", err) http.Error(wr, "Can't hijack client connection", http.StatusInternalServerError) @@ -64,12 +66,16 @@ func (s *ProxyHandler) HandleTunnel(wr http.ResponseWriter, req *http.Request) { // Inform client connection is built fmt.Fprintf(localconn, "HTTP/%d.%d 200 OK\r\n\r\n", req.ProtoMajor, req.ProtoMinor) - proxy(req.Context(), localconn, conn) + clientReader := io.Reader(localconn) + if rw != nil && rw.Reader.Buffered() > 0 { + clientReader = io.MultiReader(rw.Reader, localconn) + } + proxy(req.Context(), localconn, clientReader, conn, s.fakeSNI) } else if req.ProtoMajor == 2 { wr.Header()["Date"] = nil wr.WriteHeader(http.StatusOK) flush(wr) - proxyh2(req.Context(), req.Body, wr, conn) + proxyh2(req.Context(), req.Body, wr, conn, s.fakeSNI) } else { s.logger.Error("Unsupported protocol version: %s", req.Proto) http.Error(wr, "Unsupported protocol version.", http.StatusBadRequest) @@ -115,16 +121,21 @@ func (s *ProxyHandler) ServeHTTP(wr http.ResponseWriter, req *http.Request) { } } -func proxy(ctx context.Context, left, right net.Conn) { +func proxy(ctx context.Context, left net.Conn, leftReader io.Reader, right net.Conn, fakeSNI string) { wg := sync.WaitGroup{} - cpy := func(dst, src net.Conn) { + ltr := func(dst net.Conn, src io.Reader) { + defer wg.Done() + copyWithSNIRewrite(dst, src, fakeSNI) + dst.Close() + } + rtl := func(dst, src net.Conn) { defer wg.Done() io.Copy(dst, src) dst.Close() } wg.Add(2) - go cpy(left, right) - go cpy(right, left) + go ltr(right, leftReader) + go rtl(left, right) groupdone := make(chan struct{}) go func() { wg.Wait() @@ -141,11 +152,11 @@ func proxy(ctx context.Context, left, right net.Conn) { return } -func proxyh2(ctx context.Context, leftreader io.ReadCloser, leftwriter io.Writer, right net.Conn) { +func proxyh2(ctx context.Context, leftreader io.ReadCloser, leftwriter io.Writer, right net.Conn, fakeSNI string) { wg := sync.WaitGroup{} ltr := func(dst net.Conn, src io.Reader) { defer wg.Done() - io.Copy(dst, src) + copyWithSNIRewrite(dst, src, fakeSNI) dst.Close() } rtl := func(dst io.Writer, src io.Reader) { diff --git a/handler/socks.go b/handler/socks.go index 57037ac..e4c3a2b 100644 --- a/handler/socks.go +++ b/handler/socks.go @@ -2,14 +2,18 @@ package handler import ( "context" + "fmt" + "io" "log" "net" + "strings" "github.com/Alexey71/opera-proxy/dialer" "github.com/things-go/go-socks5" + "github.com/things-go/go-socks5/statute" ) -func NewSocksServer(dialer dialer.ContextDialer, logger *log.Logger) (*socks5.Server, error) { +func NewSocksServer(dialer dialer.ContextDialer, logger *log.Logger, fakeSNI string) (*socks5.Server, error) { opts := []socks5.Option{ socks5.WithLogger(socks5.NewLogger(logger)), socks5.WithRule( @@ -17,12 +21,45 @@ func NewSocksServer(dialer dialer.ContextDialer, logger *log.Logger) (*socks5.Se EnableConnect: true, }, ), - socks5.WithDial(dialer.DialContext), socks5.WithResolver(DummySocksResolver{}), + socks5.WithConnectHandle(func(ctx context.Context, writer io.Writer, request *socks5.Request) error { + return handleSocksConnect(ctx, writer, request, dialer, fakeSNI) + }), } return socks5.NewServer(opts...), nil } +func handleSocksConnect(ctx context.Context, writer io.Writer, request *socks5.Request, upstream dialer.ContextDialer, fakeSNI string) error { + target, err := upstream.DialContext(ctx, "tcp", request.DestAddr.String()) + if err != nil { + reply := statute.RepHostUnreachable + msg := err.Error() + if strings.Contains(msg, "refused") { + reply = statute.RepConnectionRefused + } else if strings.Contains(msg, "network is unreachable") { + reply = statute.RepNetworkUnreachable + } + if sendErr := socks5.SendReply(writer, reply, nil); sendErr != nil { + return fmt.Errorf("failed to send reply, %v", sendErr) + } + return fmt.Errorf("connect to %v failed, %v", request.RawDestAddr, err) + } + + if err := socks5.SendReply(writer, statute.RepSuccess, target.LocalAddr()); err != nil { + target.Close() + return fmt.Errorf("failed to send reply, %v", err) + } + + clientConn, ok := writer.(net.Conn) + if !ok { + target.Close() + return fmt.Errorf("writer is %T, expected net.Conn", writer) + } + + proxy(ctx, clientConn, request.Reader, target, fakeSNI) + return nil +} + type DummySocksResolver struct{} func (_ DummySocksResolver) Resolve(ctx context.Context, name string) (context.Context, net.IP, error) { diff --git a/handler/tls_sni.go b/handler/tls_sni.go new file mode 100644 index 0000000..f2dc175 --- /dev/null +++ b/handler/tls_sni.go @@ -0,0 +1,227 @@ +package handler + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + "strings" +) + +const ( + tlsRecordHeaderLen = 5 + tlsHandshakeHeaderLen = 4 + tlsRecordTypeHandshake = 0x16 + tlsHandshakeTypeClientHello = 0x01 + tlsExtensionServerName = 0x0000 +) + +func copyWithSNIRewrite(dst io.Writer, src io.Reader, fakeSNI string) error { + fakeSNI = strings.TrimSpace(fakeSNI) + if fakeSNI == "" { + _, err := io.Copy(dst, src) + return err + } + + br, ok := src.(*bufio.Reader) + if !ok { + br = bufio.NewReader(src) + } + + header, err := br.Peek(tlsRecordHeaderLen) + if err != nil { + _, copyErr := io.Copy(dst, br) + if copyErr != nil { + return copyErr + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return nil + } + return err + } + if !looksLikeTLSClientHelloRecord(header) { + _, err = io.Copy(dst, br) + return err + } + + recordLen := int(binary.BigEndian.Uint16(header[3:5])) + record := make([]byte, tlsRecordHeaderLen+recordLen) + n, err := io.ReadFull(br, record) + if err != nil { + if n > 0 { + if _, writeErr := dst.Write(record[:n]); writeErr != nil { + return writeErr + } + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + _, copyErr := io.Copy(dst, br) + return copyErr + } + return err + } + + if rewritten, ok := rewriteTLSClientHelloRecordServerName(record, fakeSNI); ok { + record = rewritten + } + + if _, err := dst.Write(record); err != nil { + return err + } + _, err = io.Copy(dst, br) + return err +} + +func looksLikeTLSClientHelloRecord(header []byte) bool { + return len(header) >= tlsRecordHeaderLen && + header[0] == tlsRecordTypeHandshake && + header[1] == 0x03 && + header[2] <= 0x04 +} + +func rewriteTLSClientHelloRecordServerName(record []byte, fakeSNI string) ([]byte, bool) { + if len(record) < tlsRecordHeaderLen+tlsHandshakeHeaderLen { + return nil, false + } + if !looksLikeTLSClientHelloRecord(record[:tlsRecordHeaderLen]) { + return nil, false + } + + payload := record[tlsRecordHeaderLen:] + if len(payload) < tlsHandshakeHeaderLen || payload[0] != tlsHandshakeTypeClientHello { + return nil, false + } + + handshakeLen := readUint24(payload[1:4]) + if handshakeLen > len(payload)-tlsHandshakeHeaderLen { + // ClientHello is fragmented across multiple TLS records. + return nil, false + } + + hello := payload[tlsHandshakeHeaderLen : tlsHandshakeHeaderLen+handshakeLen] + offset := 0 + if !skipLen(hello, &offset, 2+32) { + return nil, false + } + if !skipOpaque8(hello, &offset) { + return nil, false + } + if !skipOpaque16(hello, &offset) { + return nil, false + } + if !skipOpaque8(hello, &offset) { + return nil, false + } + if offset == len(hello) { + return nil, false + } + if offset+2 > len(hello) { + return nil, false + } + + extensionsLenOffset := offset + extensionsLen := int(binary.BigEndian.Uint16(hello[offset : offset+2])) + offset += 2 + if offset+extensionsLen > len(hello) { + return nil, false + } + + extensionsEnd := offset + extensionsLen + for offset+4 <= extensionsEnd { + extStart := offset + extType := binary.BigEndian.Uint16(hello[offset : offset+2]) + extLen := int(binary.BigEndian.Uint16(hello[offset+2 : offset+4])) + offset += 4 + if offset+extLen > extensionsEnd { + return nil, false + } + if extType != tlsExtensionServerName { + offset += extLen + continue + } + + extDataStart := offset + extDataEnd := offset + extLen + extData := hello[extDataStart:extDataEnd] + if len(extData) < 5 { + return nil, false + } + + serverNameListLen := int(binary.BigEndian.Uint16(extData[:2])) + if 2+serverNameListLen > len(extData) { + return nil, false + } + if extData[2] != 0x00 { + return nil, false + } + + nameLen := int(binary.BigEndian.Uint16(extData[3:5])) + if 5+nameLen > len(extData) { + return nil, false + } + + tail := extData[5+nameLen:] + newExtData := make([]byte, 2+1+2+len(fakeSNI)+len(tail)) + binary.BigEndian.PutUint16(newExtData[:2], uint16(1+2+len(fakeSNI)+len(tail))) + newExtData[2] = 0x00 + binary.BigEndian.PutUint16(newExtData[3:5], uint16(len(fakeSNI))) + copy(newExtData[5:], fakeSNI) + copy(newExtData[5+len(fakeSNI):], tail) + + helloStart := tlsRecordHeaderLen + tlsHandshakeHeaderLen + extLenFieldStart := helloStart + extStart + 2 + extDataAbsStart := helloStart + extDataStart + extDataAbsEnd := helloStart + extDataEnd + extensionsLenFieldStart := helloStart + extensionsLenOffset + + delta := len(newExtData) - len(extData) + newRecord := make([]byte, 0, len(record)+delta) + newRecord = append(newRecord, record[:extDataAbsStart]...) + newRecord = append(newRecord, newExtData...) + newRecord = append(newRecord, record[extDataAbsEnd:]...) + + binary.BigEndian.PutUint16(newRecord[3:5], uint16(len(payload)+delta)) + writeUint24(newRecord[6:9], handshakeLen+delta) + binary.BigEndian.PutUint16(newRecord[extensionsLenFieldStart:extensionsLenFieldStart+2], uint16(extensionsLen+delta)) + binary.BigEndian.PutUint16(newRecord[extLenFieldStart:extLenFieldStart+2], uint16(extLen+delta)) + + return newRecord, true + } + + return nil, false +} + +func readUint24(b []byte) int { + return int(b[0])<<16 | int(b[1])<<8 | int(b[2]) +} + +func writeUint24(dst []byte, v int) { + dst[0] = byte(v >> 16) + dst[1] = byte(v >> 8) + dst[2] = byte(v) +} + +func skipLen(b []byte, offset *int, n int) bool { + if *offset+n > len(b) { + return false + } + *offset += n + return true +} + +func skipOpaque8(b []byte, offset *int) bool { + if *offset >= len(b) { + return false + } + l := int(b[*offset]) + *offset++ + return skipLen(b, offset, l) +} + +func skipOpaque16(b []byte, offset *int) bool { + if *offset+2 > len(b) { + return false + } + l := int(binary.BigEndian.Uint16(b[*offset : *offset+2])) + *offset += 2 + return skipLen(b, offset, l) +} diff --git a/handler/tls_sni_test.go b/handler/tls_sni_test.go new file mode 100644 index 0000000..5d23414 --- /dev/null +++ b/handler/tls_sni_test.go @@ -0,0 +1,133 @@ +package handler + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestRewriteTLSClientHelloRecordServerName(t *testing.T) { + record := buildClientHelloRecord("example.com") + + rewritten, ok := rewriteTLSClientHelloRecordServerName(record, "fake.example") + if !ok { + t.Fatal("expected ClientHello record to be rewritten") + } + + if got := extractServerName(t, rewritten); got != "fake.example" { + t.Fatalf("unexpected SNI after rewrite: got %q", got) + } + if got := int(binary.BigEndian.Uint16(rewritten[3:5])); got != len(rewritten)-tlsRecordHeaderLen { + t.Fatalf("unexpected TLS record length: got %d want %d", got, len(rewritten)-tlsRecordHeaderLen) + } + if got := readUint24(rewritten[6:9]); got != len(rewritten)-tlsRecordHeaderLen-tlsHandshakeHeaderLen { + t.Fatalf("unexpected handshake length: got %d want %d", got, len(rewritten)-tlsRecordHeaderLen-tlsHandshakeHeaderLen) + } +} + +func TestCopyWithSNIRewritePreservesTrailingBytes(t *testing.T) { + record := buildClientHelloRecord("example.com") + stream := append(append([]byte{}, record...), []byte("tail")...) + + var dst bytes.Buffer + if err := copyWithSNIRewrite(&dst, bytes.NewReader(stream), "fake.example"); err != nil { + t.Fatalf("copyWithSNIRewrite returned error: %v", err) + } + + out := dst.Bytes() + recordLen := int(binary.BigEndian.Uint16(out[3:5])) + tlsRecordHeaderLen + if got := extractServerName(t, out[:recordLen]); got != "fake.example" { + t.Fatalf("unexpected SNI in output stream: got %q", got) + } + if tail := string(out[recordLen:]); tail != "tail" { + t.Fatalf("unexpected trailing bytes: got %q", tail) + } +} + +func TestRewriteTLSClientHelloRecordServerNameSkipsFragmentedHello(t *testing.T) { + record := buildClientHelloRecord("example.com") + record[6] = 0x7f + record[7] = 0xff + record[8] = 0xff + + if _, ok := rewriteTLSClientHelloRecordServerName(record, "fake.example"); ok { + t.Fatal("expected fragmented ClientHello rewrite to be skipped") + } +} + +func buildClientHelloRecord(serverName string) []byte { + sniExtData := make([]byte, 2+1+2+len(serverName)) + binary.BigEndian.PutUint16(sniExtData[:2], uint16(1+2+len(serverName))) + sniExtData[2] = 0x00 + binary.BigEndian.PutUint16(sniExtData[3:5], uint16(len(serverName))) + copy(sniExtData[5:], serverName) + + sniExt := makeExtension(tlsExtensionServerName, sniExtData) + otherExt := makeExtension(0x002b, []byte{0x02, 0x03, 0x04}) + extensions := append(sniExt, otherExt...) + + hello := make([]byte, 0, 128) + hello = append(hello, 0x03, 0x03) + hello = append(hello, bytes.Repeat([]byte{0x11}, 32)...) + hello = append(hello, 0x00) + hello = append(hello, 0x00, 0x02, 0x13, 0x01) + hello = append(hello, 0x01, 0x00) + hello = append(hello, byte(len(extensions)>>8), byte(len(extensions))) + hello = append(hello, extensions...) + + record := make([]byte, 0, tlsRecordHeaderLen+tlsHandshakeHeaderLen+len(hello)) + record = append(record, tlsRecordTypeHandshake, 0x03, 0x03) + recordLen := tlsHandshakeHeaderLen + len(hello) + record = append(record, byte(recordLen>>8), byte(recordLen)) + record = append(record, tlsHandshakeTypeClientHello) + record = append(record, byte(len(hello)>>16), byte(len(hello)>>8), byte(len(hello))) + record = append(record, hello...) + return record +} + +func makeExtension(extType uint16, data []byte) []byte { + ext := make([]byte, 4+len(data)) + binary.BigEndian.PutUint16(ext[:2], extType) + binary.BigEndian.PutUint16(ext[2:4], uint16(len(data))) + copy(ext[4:], data) + return ext +} + +func extractServerName(t *testing.T, record []byte) string { + t.Helper() + + payload := record[tlsRecordHeaderLen:] + handshakeLen := readUint24(payload[1:4]) + hello := payload[tlsHandshakeHeaderLen : tlsHandshakeHeaderLen+handshakeLen] + + offset := 2 + 32 + sessionIDLen := int(hello[offset]) + offset++ + offset += sessionIDLen + + cipherSuitesLen := int(binary.BigEndian.Uint16(hello[offset : offset+2])) + offset += 2 + cipherSuitesLen + + compressionMethodsLen := int(hello[offset]) + offset++ + offset += compressionMethodsLen + + extensionsLen := int(binary.BigEndian.Uint16(hello[offset : offset+2])) + offset += 2 + extensionsEnd := offset + extensionsLen + for offset+4 <= extensionsEnd { + extType := binary.BigEndian.Uint16(hello[offset : offset+2]) + extLen := int(binary.BigEndian.Uint16(hello[offset+2 : offset+4])) + offset += 4 + if extType != tlsExtensionServerName { + offset += extLen + continue + } + extData := hello[offset : offset+extLen] + nameLen := int(binary.BigEndian.Uint16(extData[3:5])) + return string(extData[5 : 5+nameLen]) + } + + t.Fatal("server_name extension not found") + return "" +} diff --git a/main.go b/main.go index 43b6676..43de79b 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "context" "crypto/tls" @@ -17,8 +18,10 @@ import ( "net/url" "os" "runtime/debug" + "sort" "strconv" "strings" + "sync" "time" xproxy "golang.org/x/net/proxy" @@ -35,8 +38,10 @@ import ( ) const ( - API_DOMAIN = "api2.sec-tunnel.com" - PROXY_SUFFIX = "sec-tunnel.com" + API_DOMAIN = "api2.sec-tunnel.com" + PROXY_SUFFIX = "sec-tunnel.com" + DefaultDiscoverCSVFallback = "proxies.csv" + DefaultProxyBypassFallback = "proxy-bypass.txt" ) func perror(msg string) { @@ -101,9 +106,18 @@ func (a *serverSelectionArg) String() string { type CLIArgs struct { country string + countryExplicit bool + discoverCSV string listCountries bool listProxies bool + listProxiesAll bool + listProxiesAllOut string + fetchFreeProxyOut string + fetchFreeProxyURL string + estimateProxySpeed bool + sortProxiesBy string dpExport bool + discoverRepeat int bindAddress string socksMode bool verbosity int @@ -117,6 +131,9 @@ type CLIArgs struct { apiClientVersion string apiUserAgent string apiProxy string + apiProxyFile string + apiProxyListURL string + apiProxyParallel int bootstrapDNS *CSVArg refresh time.Duration refreshRetry time.Duration @@ -124,7 +141,13 @@ type CLIArgs struct { initRetryInterval time.Duration caFile string fakeSNI string + proxyBypass *CSVArg + proxyBlacklistFile string + proxyBlacklist map[string]struct{} overrideProxyAddress string + proxySpeedTestURL string + proxySpeedTimeout time.Duration + proxySpeedDLLimit int64 serverSelection serverSelectionArg serverSelectionTimeout time.Duration serverSelectionTestURL string @@ -146,12 +169,21 @@ func parse_args() *CLIArgs { "https://doh.cleanbrowsing.org/doh/adult-filter/", }, }, + proxyBypass: &CSVArg{}, serverSelection: serverSelectionArg{dialer.ServerSelectionFastest}, } - flag.StringVar(&args.country, "country", "EU", "desired proxy location") + flag.StringVar(&args.country, "country", "EU", "desired proxy location; for list-proxies-all modes supports comma-separated codes or ALL") + flag.StringVar(&args.discoverCSV, "discover-csv", "", "read proxy endpoints from CSV instead of SurfEasy discover API") flag.BoolVar(&args.listCountries, "list-countries", false, "list available countries and exit") flag.BoolVar(&args.listProxies, "list-proxies", false, "output proxy list and exit") + flag.BoolVar(&args.listProxiesAll, "list-proxies-all", false, "output proxy list for all countries and exit") + flag.StringVar(&args.listProxiesAllOut, "list-proxies-all-out", "", "write proxy list CSV to file") + flag.StringVar(&args.fetchFreeProxyOut, "fetch-freeproxy-out", "", "download proxy list from advanced.name/freeproxy and save it as a text file with one ip:port per line") + flag.StringVar(&args.fetchFreeProxyURL, "fetch-freeproxy-url", "https://advanced.name/freeproxy", "source URL for -fetch-freeproxy-out") + flag.BoolVar(&args.estimateProxySpeed, "estimate-proxy-speed", false, "measure proxy response time for proxy list output") + flag.StringVar(&args.sortProxiesBy, "sort-proxies-by", "speed", "proxy list sort order: speed, country, ip") flag.BoolVar(&args.dpExport, "dp-export", false, "export configuration for dumbproxy") + flag.IntVar(&args.discoverRepeat, "discover-repeat", 1, "number of repeated discover requests to aggregate and deduplicate") flag.StringVar(&args.bindAddress, "bind-address", "127.0.0.1:18080", "proxy listen address") flag.BoolVar(&args.socksMode, "socks-mode", false, "listen for SOCKS requests instead of HTTP") flag.IntVar(&args.verbosity, "verbosity", 20, "logging verbosity "+ @@ -161,6 +193,7 @@ func parse_args() *CLIArgs { flag.StringVar(&args.proxy, "proxy", "", "sets base proxy to use for all dial-outs. "+ "Format: ://[login:password@]host[:port] "+ "Examples: http://user:password@192.168.1.1:3128, socks5://10.0.0.1:1080") + flag.Var(args.proxyBypass, "proxy-bypass", "comma-separated list of destination host or URL patterns that should bypass proxying and connect directly; matching is case-insensitive and supports * in hostnames") flag.StringVar(&args.apiClientVersion, "api-client-version", se.DefaultSESettings.ClientVersion, "client version reported to SurfEasy API") flag.StringVar(&args.apiClientType, "api-client-type", se.DefaultSESettings.ClientType, "client type reported to SurfEasy API") flag.StringVar(&args.apiUserAgent, "api-user-agent", se.DefaultSESettings.UserAgent, "user agent reported to SurfEasy API") @@ -168,6 +201,9 @@ func parse_args() *CLIArgs { flag.StringVar(&args.apiPassword, "api-password", "SILrMEPBmJuhomxWkfm3JalqHX2Eheg1YhlEZiMh8II", "SurfEasy API password") flag.StringVar(&args.apiAddress, "api-address", "", fmt.Sprintf("override IP address of %s", API_DOMAIN)) flag.StringVar(&args.apiProxy, "api-proxy", "", "additional proxy server used to access SurfEasy API") + flag.StringVar(&args.apiProxyFile, "api-proxy-file", "", "path to text file with candidate proxy servers for SurfEasy API access, one per line; proxies are tried in order until init/discover succeeds") + flag.StringVar(&args.apiProxyListURL, "api-proxy-list-url", "", "URL of a text file with candidate proxy servers for SurfEasy API access; falls back to -api-proxy-file if download fails") + flag.IntVar(&args.apiProxyParallel, "api-proxy-parallel", 5, "number of API proxy candidates tested in parallel when -api-proxy-file is used") flag.Var(args.bootstrapDNS, "bootstrap-dns", "comma-separated list of DNS/DoH/DoT resolvers for initial discovery of SurfEasy API address. "+ "Supported schemes are: dns://, https://, tls://, tcp://. "+ @@ -177,8 +213,13 @@ func parse_args() *CLIArgs { flag.IntVar(&args.initRetries, "init-retries", 0, "number of attempts for initialization steps, zero for unlimited retry") flag.DurationVar(&args.initRetryInterval, "init-retry-interval", 5*time.Second, "delay between initialization retries") flag.StringVar(&args.caFile, "cafile", "", "use custom CA certificate bundle file") - flag.StringVar(&args.fakeSNI, "fake-SNI", "", "domain name to use as SNI in communications with servers") + flag.StringVar(&args.fakeSNI, "fake-SNI", "", "domain name to use as SNI in outbound TLS and tunneled TLS ClientHello where possible") + flag.StringVar(&args.proxyBlacklistFile, "proxy-blacklist", "", "path to file with blacklisted proxy addresses, one host[:port] per line") flag.StringVar(&args.overrideProxyAddress, "override-proxy-address", "", "use fixed proxy address instead of server address returned by SurfEasy API") + flag.StringVar(&args.proxySpeedTestURL, "proxy-speed-test-url", "https://ajax.googleapis.com/ajax/libs/angularjs/1.8.2/angular.min.js", + "URL used to measure proxy response time") + flag.DurationVar(&args.proxySpeedTimeout, "proxy-speed-timeout", 15*time.Second, "timeout for a single proxy speed measurement") + flag.Int64Var(&args.proxySpeedDLLimit, "proxy-speed-dl-limit", 262144, "limit of downloaded bytes for proxy speed measurement") flag.Var(&args.serverSelection, "server-selection", "server selection policy (first/random/fastest)") flag.DurationVar(&args.serverSelectionTimeout, "server-selection-timeout", 30*time.Second, "timeout given for server selection function to produce result") flag.StringVar(&args.serverSelectionTestURL, "server-selection-test-url", "https://ajax.googleapis.com/ajax/libs/angularjs/1.8.2/angular.min.js", @@ -186,10 +227,41 @@ func parse_args() *CLIArgs { flag.Int64Var(&args.serverSelectionDLLimit, "server-selection-dl-limit", 0, "restrict amount of downloaded data per connection by fastest server selection") flag.Func("config", "read configuration from file with space-separated keys and values", readConfig) flag.Parse() + flag.Visit(func(f *flag.Flag) { + if f.Name == "country" { + args.countryExplicit = true + } + }) if args.country == "" { arg_fail("Country can't be empty string.") } - if args.listCountries && args.listProxies || args.listCountries && args.dpExport || args.listProxies && args.dpExport { + if args.discoverRepeat < 1 { + arg_fail("discover-repeat must be >= 1.") + } + if args.apiProxyParallel < 1 { + arg_fail("api-proxy-parallel must be >= 1.") + } + switch args.sortProxiesBy { + case "speed", "country", "ip": + default: + arg_fail("sort-proxies-by must be one of: speed, country, ip.") + } + if args.listProxiesAllOut != "" { + args.listProxiesAll = true + } + if args.listProxiesAll { + args.estimateProxySpeed = true + } + if args.listCountries && args.listProxies || + args.listCountries && args.listProxiesAll || + args.listCountries && args.dpExport || + args.listCountries && args.fetchFreeProxyOut != "" || + args.listProxies && args.listProxiesAll || + args.listProxies && args.dpExport || + args.listProxies && args.fetchFreeProxyOut != "" || + args.listProxiesAll && args.dpExport || + args.listProxiesAll && args.fetchFreeProxyOut != "" || + args.dpExport && args.fetchFreeProxyOut != "" { arg_fail("mutually exclusive output arguments were provided") } return args @@ -204,12 +276,359 @@ func proxyFromURLWrapper(u *url.URL, next xproxy.Dialer) (xproxy.Dialer, error) return dialer.ProxyDialerFromURL(u, cdialer) } +func normalizeAPIProxy(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", nil + } + if strings.Contains(raw, "://") { + return raw, nil + } + if strings.Contains(raw, "@") { + return "http://" + raw, nil + } + + parts := strings.Split(raw, ":") + if len(parts) == 4 { + host := strings.TrimSpace(parts[0]) + port := strings.TrimSpace(parts[1]) + user := strings.TrimSpace(parts[2]) + password := strings.TrimSpace(parts[3]) + if host == "" || port == "" || user == "" { + return "", fmt.Errorf("invalid proxy entry %q", raw) + } + proxyURL := &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(host, port), + User: url.UserPassword(user, password), + } + return proxyURL.String(), nil + } + + if len(parts) == 2 { + host := strings.TrimSpace(parts[0]) + port := strings.TrimSpace(parts[1]) + if host == "" || port == "" { + return "", fmt.Errorf("invalid proxy entry %q", raw) + } + return "http://" + net.JoinHostPort(host, port), nil + } + + return "", fmt.Errorf("unsupported proxy entry format %q", raw) +} + +func loadAPIProxyListFromReader(r io.Reader, source string) ([]string, error) { + scanner := bufio.NewScanner(r) + proxies := make([]string, 0) + seen := make(map[string]struct{}) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + proxy, err := normalizeAPIProxy(line) + if err != nil { + return nil, fmt.Errorf("invalid API proxy entry %q from %s: %w", line, source, err) + } + if _, ok := seen[proxy]; ok { + continue + } + seen[proxy] = struct{}{} + proxies = append(proxies, proxy) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("unable to read API proxy list from %s: %w", source, err) + } + if len(proxies) == 0 { + return nil, fmt.Errorf("no API proxies found in %s", source) + } + return proxies, nil +} + +func loadAPIProxyList(filename string) ([]string, error) { + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("unable to open API proxy list %q: %w", filename, err) + } + defer f.Close() + return loadAPIProxyListFromReader(f, fmt.Sprintf("file %q", filename)) +} + +func loadProxyBypassList(filename string) ([]string, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + seen := make(map[string]struct{}) + patterns := make([]string, 0) + for scanner.Scan() { + line := scanner.Text() + if idx := strings.Index(line, "#"); idx >= 0 { + line = line[:idx] + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + + var parsed CSVArg + if err := parsed.Set(line); err != nil { + return nil, fmt.Errorf("unable to parse proxy bypass entry %q from %s: %w", line, filename, err) + } + for _, value := range parsed.values { + pattern := strings.TrimSpace(value) + if pattern == "" { + continue + } + if _, ok := seen[pattern]; ok { + continue + } + seen[pattern] = struct{}{} + patterns = append(patterns, pattern) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("unable to read proxy bypass list %q: %w", filename, err) + } + return patterns, nil +} + +func loadAPIProxyListFromURL(listURL string, transport http.RoundTripper, timeout time.Duration) ([]string, error) { + client := &http.Client{ + Transport: transport, + Timeout: timeout, + } + req, err := http.NewRequest(http.MethodGet, listURL, nil) + if err != nil { + return nil, fmt.Errorf("unable to create request for API proxy list URL %q: %w", listURL, err) + } + req.Header.Set("User-Agent", "opera-proxy api-proxy-list fetcher/1.0") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to download API proxy list from %q: %w", listURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unable to download API proxy list from %q: unexpected HTTP status %s", listURL, resp.Status) + } + return loadAPIProxyListFromReader(resp.Body, fmt.Sprintf("URL %q", listURL)) +} + +func newSEClient(args *CLIArgs, baseDialer dialer.ContextDialer, caPool *x509.CertPool, apiProxy string) (*se.SEClient, error) { + seclientDialer := baseDialer + if apiProxy != "" { + apiProxyURL, err := url.Parse(apiProxy) + if err != nil { + return nil, fmt.Errorf("unable to parse API proxy URL %q: %w", apiProxy, err) + } + pxDialer, err := xproxy.FromURL(apiProxyURL, seclientDialer) + if err != nil { + return nil, fmt.Errorf("unable to instantiate API proxy dialer %q: %w", apiProxy, err) + } + seclientDialer = pxDialer.(dialer.ContextDialer) + } + if args.apiAddress != "" { + seclientDialer = dialer.NewFixedDialer(args.apiAddress, seclientDialer) + } else if len(args.bootstrapDNS.values) > 0 { + resolver, err := resolver.FastFromURLs(caPool, args.bootstrapDNS.values...) + if err != nil { + return nil, fmt.Errorf("unable to instantiate DNS resolver: %w", err) + } + seclientDialer = dialer.NewResolvingDialer(resolver, seclientDialer) + } + + // Dialing w/o SNI, receiving self-signed certificate, so skip verification. + // Either way we'll validate certificate of actual proxy server. + tlsConfig := &tls.Config{ + ServerName: args.fakeSNI, + InsecureSkipVerify: true, + } + seclient, err := se.NewSEClient(args.apiLogin, args.apiPassword, &http.Transport{ + DialContext: seclientDialer.DialContext, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := seclientDialer.DialContext(ctx, network, addr) + if err != nil { + return conn, err + } + return tls.Client(conn, tlsConfig), nil + }, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("unable to construct SEClient: %w", err) + } + seclient.Settings.ClientType = args.apiClientType + seclient.Settings.ClientVersion = args.apiClientVersion + seclient.Settings.UserAgent = args.apiUserAgent + return seclient, nil +} + +type apiProxyCandidateResult struct { + candidate string + client *se.SEClient + ips []se.SEIPEntry + countries []se.SEGeoEntry + err error +} + +func callWithTimeout(parent context.Context, timeout time.Duration, f func(context.Context) error) error { + ctx, cl := context.WithTimeout(parent, timeout) + defer cl() + return f(ctx) +} + +func testAPIProxyCandidate(ctx context.Context, args *CLIArgs, baseDialer dialer.ContextDialer, caPool *x509.CertPool, candidate string, needDiscover bool, needCountries bool) apiProxyCandidateResult { + result := apiProxyCandidateResult{candidate: candidate} + + client, err := newSEClient(args, baseDialer, caPool, candidate) + if err != nil { + result.err = err + return result + } + + if err := callWithTimeout(ctx, args.timeout, client.AnonRegister); err != nil { + result.err = fmt.Errorf("anonymous registration failed: %w", err) + return result + } + + if err := callWithTimeout(ctx, args.timeout, client.RegisterDevice); err != nil { + result.err = fmt.Errorf("device registration failed: %w", err) + return result + } + + if needCountries { + var countries []se.SEGeoEntry + if err := callWithTimeout(ctx, args.timeout, func(reqCtx context.Context) error { + var geoErr error + countries, geoErr = client.GeoList(reqCtx) + return geoErr + }); err != nil { + result.err = fmt.Errorf("geo list request failed: %w", err) + return result + } + result.countries = countries + } + + if needDiscover { + var ips []se.SEIPEntry + if args.listProxiesAll { + ips, err = discoverAllCountriesWithContext(ctx, args, client, nil, args.country) + } else { + ips, err = discoverCountryWithContext(ctx, args, client, nil, args.country) + } + if err != nil { + result.err = fmt.Errorf("discover request failed: %w", err) + return result + } + if len(ips) == 0 { + result.err = errors.New("discover request returned an empty endpoints list") + return result + } + result.ips = ips + } + + result.client = client + return result +} + +func selectAPIProxyCandidate(args *CLIArgs, baseDialer dialer.ContextDialer, caPool *x509.CertPool, logger *clog.CondLogger, candidates []string, needDiscover bool, needCountries bool) (apiProxyCandidateResult, error) { + if len(candidates) == 0 { + return apiProxyCandidateResult{}, errors.New("no API proxy candidates provided") + } + + parallelism := args.apiProxyParallel + if parallelism > len(candidates) { + parallelism = len(candidates) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + type candidateJob struct { + idx int + candidate string + } + + jobs := make(chan candidateJob) + results := make(chan apiProxyCandidateResult, len(candidates)) + var wg sync.WaitGroup + + worker := func() { + defer wg.Done() + for job := range jobs { + if ctx.Err() != nil { + return + } + logger.Info("Trying API proxy candidate #%d/%d: %s", job.idx+1, len(candidates), job.candidate) + result := testAPIProxyCandidate(ctx, args, baseDialer, caPool, job.candidate, needDiscover, needCountries) + select { + case results <- result: + case <-ctx.Done(): + return + } + } + } + + wg.Add(parallelism) + for i := 0; i < parallelism; i++ { + go worker() + } + + go func() { + defer close(jobs) + for idx, candidate := range candidates { + select { + case jobs <- candidateJob{idx: idx, candidate: candidate}: + case <-ctx.Done(): + return + } + } + }() + + var lastErr error + for i := 0; i < len(candidates); i++ { + result := <-results + if result.err == nil { + logger.Info("Using API proxy candidate: %s", result.candidate) + cancel() + wg.Wait() + return result, nil + } + lastErr = result.err + logger.Warning("API proxy candidate %s failed: %v", result.candidate, result.err) + } + + cancel() + wg.Wait() + if lastErr == nil { + lastErr = errors.New("all API proxy candidates failed") + } + return apiProxyCandidateResult{}, lastErr +} + func run() int { args := parse_args() if args.showVersion { fmt.Println(version()) return 0 } + if args.fetchFreeProxyOut != "" { + count, err := fetchFreeProxyToFile(args.fetchFreeProxyURL, args.fetchFreeProxyOut, args.timeout) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to fetch free proxy list: %v\n", err) + return 19 + } + fmt.Printf("Saved %d proxies to %s\n", count, args.fetchFreeProxyOut) + return 0 + } logWriter := clog.NewLogWriter(os.Stderr) defer logWriter.Close() @@ -225,6 +644,16 @@ func run() int { mainLogger.Info("opera-proxy client version %s is starting...", version()) + proxyBlacklist, err := loadProxyBlacklist(args.proxyBlacklistFile) + if err != nil { + mainLogger.Error("Can't load proxy blacklist file %q: %v", args.proxyBlacklistFile, err) + return 18 + } + args.proxyBlacklist = proxyBlacklist + if len(args.proxyBlacklist) > 0 { + mainLogger.Info("Loaded %d blacklisted proxy endpoints from %s.", len(args.proxyBlacklist), args.proxyBlacklistFile) + } + var d dialer.ContextDialer = &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -258,6 +687,7 @@ func run() int { xproxy.RegisterDialerType("http", proxyFromURLWrapper) xproxy.RegisterDialerType("https", proxyFromURLWrapper) + directDialer := d if args.proxy != "" { proxyURL, err := url.Parse(args.proxy) if err != nil { @@ -271,115 +701,182 @@ func run() int { } d = pxDialer.(dialer.ContextDialer) } - - seclientDialer := d - if args.apiProxy != "" { - apiProxyURL, err := url.Parse(args.apiProxy) - if err != nil { - mainLogger.Critical("Unable to parse base proxy URL: %v", err) - return 6 - } - pxDialer, err := xproxy.FromURL(apiProxyURL, seclientDialer) - if err != nil { - mainLogger.Critical("Unable to instantiate base proxy dialer: %v", err) + if len(args.proxyBypass.values) == 0 { + fallbackPatterns, err := loadProxyBypassList(proxyBypassPathForFallback()) + if err == nil { + args.proxyBypass.values = fallbackPatterns + } else if !errors.Is(err, os.ErrNotExist) { + mainLogger.Critical("Unable to load proxy bypass list: %v", err) return 7 } - seclientDialer = pxDialer.(dialer.ContextDialer) } - if args.apiAddress != "" { - mainLogger.Info("Using fixed API host address = %s", args.apiAddress) - seclientDialer = dialer.NewFixedDialer(args.apiAddress, seclientDialer) - } else if len(args.bootstrapDNS.values) > 0 { - resolver, err := resolver.FastFromURLs(caPool, args.bootstrapDNS.values...) - if err != nil { - mainLogger.Critical("Unable to instantiate DNS resolver: %v", err) - return 4 - } - seclientDialer = dialer.NewResolvingDialer(resolver, seclientDialer) - } - - // Dialing w/o SNI, receiving self-signed certificate, so skip verification. - // Either way we'll validate certificate of actual proxy server. - tlsConfig := &tls.Config{ - ServerName: args.fakeSNI, - InsecureSkipVerify: true, - } - seclient, err := se.NewSEClient(args.apiLogin, args.apiPassword, &http.Transport{ - DialContext: seclientDialer.DialContext, - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - conn, err := seclientDialer.DialContext(ctx, network, addr) - if err != nil { - return conn, err - } - return tls.Client(conn, tlsConfig), nil - }, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }) - if err != nil { - mainLogger.Critical("Unable to construct SEClient: %v", err) - return 8 - } - seclient.Settings.ClientType = args.apiClientType - seclient.Settings.ClientVersion = args.apiClientVersion - seclient.Settings.UserAgent = args.apiUserAgent + mainLogger.Info("Proxy bypass loaded: %d rule(s).", len(args.proxyBypass.values)) try := retryPolicy(args.initRetries, args.initRetryInterval, mainLogger) - - err = try("anonymous registration", func() error { - ctx, cl := context.WithTimeout(context.Background(), args.timeout) - defer cl() - return seclient.AnonRegister(ctx) - }) - if err != nil { - return 9 + if args.apiAddress != "" { + mainLogger.Info("Using fixed API host address = %s", args.apiAddress) } - err = try("device registration", func() error { - ctx, cl := context.WithTimeout(context.Background(), args.timeout) - defer cl() - return seclient.RegisterDevice(ctx) - }) - if err != nil { - return 10 - } + var ( + seclient *se.SEClient + ips []se.SEIPEntry + preloadedDiscovery bool + ) - if args.listCountries { - return printCountries(try, mainLogger, args.timeout, seclient) - } - - var ips []se.SEIPEntry - if args.listProxies || args.dpExport { - err = try("discover", func() error { - ctx, cl := context.WithTimeout(context.Background(), args.timeout) - defer cl() - ips, err = seclient.Discover(ctx, fmt.Sprintf("\"%s\",,", args.country)) + if args.apiProxyListURL != "" || args.apiProxyFile != "" { + var ( + candidates []string + err error + ) + if args.apiProxyListURL != "" { + downloadTransport := &http.Transport{ + DialContext: d.DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + candidates, err = loadAPIProxyListFromURL(args.apiProxyListURL, downloadTransport, args.timeout) if err != nil { - return err + if args.apiProxyFile == "" { + mainLogger.Critical("Unable to load API proxy list from URL: %v", err) + return 8 + } + mainLogger.Warning("Unable to load API proxy list from URL %q, falling back to file %q: %v", args.apiProxyListURL, args.apiProxyFile, err) } - if len(ips) == 0 { - return errors.New("empty endpoints list!") + } + if len(candidates) == 0 { + candidates, err = loadAPIProxyList(args.apiProxyFile) + if err != nil { + mainLogger.Critical("Unable to load API proxy list: %v", err) + return 8 } - return nil - }) + } + if args.apiProxy != "" { + explicitProxy, err := normalizeAPIProxy(args.apiProxy) + if err != nil { + mainLogger.Critical("Unable to parse explicit API proxy: %v", err) + return 6 + } + filtered := make([]string, 0, len(candidates)+1) + filtered = append(filtered, explicitProxy) + for _, candidate := range candidates { + if candidate == explicitProxy { + continue + } + filtered = append(filtered, candidate) + } + candidates = filtered + } + parallelism := args.apiProxyParallel + if parallelism > len(candidates) { + parallelism = len(candidates) + } + mainLogger.Info("Loaded %d API proxy candidates. Testing up to %d in parallel.", len(candidates), parallelism) + + needDiscover := args.listProxies || args.listProxiesAll || args.dpExport || args.overrideProxyAddress == "" + result, err := selectAPIProxyCandidate(args, d, caPool, mainLogger, candidates, needDiscover, args.listCountries) if err != nil { + mainLogger.Critical("All API proxy candidates failed. Last error: %v", err) return 12 } - if args.listProxies { - return printProxies(ips, seclient) + seclient = result.client + ips = result.ips + preloadedDiscovery = len(ips) > 0 + args.apiProxy = result.candidate + if args.listCountries { + return printCountryList(result.countries) } - if args.dpExport { - return dpExport(ips, seclient, args.fakeSNI) + } else { + seclientDialer := d + if args.apiProxy != "" { + apiProxyURL, err := url.Parse(args.apiProxy) + if err != nil { + mainLogger.Critical("Unable to parse base proxy URL: %v", err) + return 6 + } + pxDialer, err := xproxy.FromURL(apiProxyURL, seclientDialer) + if err != nil { + mainLogger.Critical("Unable to instantiate base proxy dialer: %v", err) + return 7 + } + seclientDialer = pxDialer.(dialer.ContextDialer) + } + if args.apiAddress != "" { + seclientDialer = dialer.NewFixedDialer(args.apiAddress, seclientDialer) + } else if len(args.bootstrapDNS.values) > 0 { + resolver, err := resolver.FastFromURLs(caPool, args.bootstrapDNS.values...) + if err != nil { + mainLogger.Critical("Unable to instantiate DNS resolver: %v", err) + return 4 + } + seclientDialer = dialer.NewResolvingDialer(resolver, seclientDialer) + } + + // Dialing w/o SNI, receiving self-signed certificate, so skip verification. + // Either way we'll validate certificate of actual proxy server. + tlsConfig := &tls.Config{ + ServerName: args.fakeSNI, + InsecureSkipVerify: true, + } + seclient, err = se.NewSEClient(args.apiLogin, args.apiPassword, &http.Transport{ + DialContext: seclientDialer.DialContext, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := seclientDialer.DialContext(ctx, network, addr) + if err != nil { + return conn, err + } + return tls.Client(conn, tlsConfig), nil + }, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }) + if err != nil { + mainLogger.Critical("Unable to construct SEClient: %v", err) + return 8 + } + seclient.Settings.ClientType = args.apiClientType + seclient.Settings.ClientVersion = args.apiClientVersion + seclient.Settings.UserAgent = args.apiUserAgent + + err = try("anonymous registration", func() error { + ctx, cl := context.WithTimeout(context.Background(), args.timeout) + defer cl() + return seclient.AnonRegister(ctx) + }) + if err != nil { + return 9 + } + + err = try("device registration", func() error { + ctx, cl := context.WithTimeout(context.Background(), args.timeout) + defer cl() + return seclient.RegisterDevice(ctx) + }) + if err != nil { + return 10 + } + + if args.listCountries { + return printCountries(try, mainLogger, args.timeout, seclient) } } - handlerDialerFactory := func(endpointAddr string) dialer.ContextDialer { + proxyTLSServerName := func(entry se.SEIPEntry) string { + if strings.TrimSpace(entry.Host) != "" { + return entry.Host + } + return fmt.Sprintf("%s0.%s", strings.ToLower(entry.Geo.CountryCode), PROXY_SUFFIX) + } + + handlerDialerFactory := func(entry se.SEIPEntry, endpointAddr string) dialer.ContextDialer { return dialer.NewProxyDialer( dialer.WrapStringToCb(endpointAddr), - dialer.WrapStringToCb(fmt.Sprintf("%s0.%s", args.country, PROXY_SUFFIX)), + dialer.WrapStringToCb(proxyTLSServerName(entry)), dialer.WrapStringToCb(args.fakeSNI), func() (string, error) { return dialer.BasicAuthHeader(seclient.GetProxyCredentials()), nil @@ -388,21 +885,70 @@ func run() int { d) } + if args.listProxies || args.listProxiesAll || args.dpExport { + if !preloadedDiscovery { + err = try("discover", func() error { + var discoverErr error + if args.listProxiesAll { + ips, discoverErr = discoverAllCountries(args, seclient, mainLogger) + } else { + ips, discoverErr = discoverCountry(args, seclient, mainLogger, args.country) + } + if discoverErr != nil { + return discoverErr + } + if len(ips) == 0 { + return errors.New("empty endpoints list!") + } + return nil + }) + if err != nil { + return 12 + } + } + if args.listProxies || args.listProxiesAll { + var speedResults map[proxyEndpointKey]proxySpeedResult + if args.estimateProxySpeed { + mainLogger.Info("Measuring proxy response time for %d endpoints using %q.", countProxyPorts(ips), args.proxySpeedTestURL) + speedResults = benchmarkProxyEndpoints(args, ips, caPool, mainLogger, handlerDialerFactory) + } + if args.listProxiesAllOut != "" { + if err := writeProxyCSV(args.listProxiesAllOut, ips, seclient, speedResults, args.sortProxiesBy); err != nil { + mainLogger.Critical("Unable to write proxy CSV: %v", err) + return 17 + } + fmt.Printf("Proxy list saved to %s\n", args.listProxiesAllOut) + return 0 + } + return printProxies(ips, seclient, speedResults, args.sortProxiesBy) + } + if args.dpExport { + return dpExport(ips, seclient, args.fakeSNI) + } + } + var handlerDialer dialer.ContextDialer if args.overrideProxyAddress == "" { - err = try("discover", func() error { - ctx, cl := context.WithTimeout(context.Background(), args.timeout) - defer cl() - res, err := seclient.Discover(ctx, fmt.Sprintf("\"%s\",,", args.country)) + if !preloadedDiscovery { + err = try("discover", func() error { + res, err := discoverCountry(args, seclient, mainLogger, args.country) + if err != nil { + return err + } + if len(res) == 0 { + return errors.New("empty endpoints list!") + } + ips = res + return nil + }) if err != nil { - return err - } - if len(res) == 0 { - return errors.New("empty endpoints list!") + return 12 } + } - mainLogger.Info("Discovered endpoints: %v. Starting server selection routine %q.", res, args.serverSelection.value) + mainLogger.Info("Discovered endpoints: %v. Starting server selection routine %q.", ips, args.serverSelection.value) + err = func() error { var ss dialer.SelectionFunc switch args.serverSelection.value { case dialer.ServerSelectionFirst: @@ -420,11 +966,11 @@ func run() int { default: panic("unhandled server selection value got past parsing") } - dialers := make([]dialer.ContextDialer, len(res)) - for i, ep := range res { - dialers[i] = handlerDialerFactory(ep.NetAddr()) + dialers := make([]dialer.ContextDialer, len(ips)) + for i, ep := range ips { + dialers[i] = handlerDialerFactory(ep, ep.NetAddr()) } - ctx, cl = context.WithTimeout(context.Background(), args.serverSelectionTimeout) + ctx, cl := context.WithTimeout(context.Background(), args.serverSelectionTimeout) defer cl() handlerDialer, err = ss(ctx, dialers) if err != nil { @@ -436,15 +982,31 @@ func run() int { } } return nil - }) + }() if err != nil { return 12 } } else { sanitizedEndpoint := sanitizeFixedProxyAddress(args.overrideProxyAddress) - handlerDialer = handlerDialerFactory(sanitizedEndpoint) + if _, ok := args.proxyBlacklist[sanitizedEndpoint]; ok { + mainLogger.Critical("Endpoint override %s is blacklisted.", sanitizedEndpoint) + return 12 + } + handlerDialer = handlerDialerFactory(se.SEIPEntry{ + Geo: se.SEGeoEntry{ + CountryCode: args.country, + }, + }, sanitizedEndpoint) mainLogger.Info("Endpoint override: %s", sanitizedEndpoint) } + if len(args.proxyBypass.values) > 0 { + bypassDialer, err := dialer.NewBypassDialer(args.proxyBypass.values, directDialer, handlerDialer) + if err != nil { + mainLogger.Critical("Unable to configure proxy bypass rules: %v", err) + return 7 + } + handlerDialer = bypassDialer + } clock.RunTicker(context.Background(), args.refresh, args.refreshRetry, func(ctx context.Context) error { mainLogger.Info("Refreshing login...") @@ -471,7 +1033,7 @@ func run() int { mainLogger.Info("Starting proxy server...") if args.socksMode { - socks, initError := handler.NewSocksServer(handlerDialer, socksLogger) + socks, initError := handler.NewSocksServer(handlerDialer, socksLogger, args.fakeSNI) if initError != nil { mainLogger.Critical("Failed to start: %v", err) return 16 @@ -479,7 +1041,7 @@ func run() int { mainLogger.Info("Init complete.") err = socks.ListenAndServe("tcp", args.bindAddress) } else { - h := handler.NewProxyHandler(handlerDialer, proxyLogger) + h := handler.NewProxyHandler(handlerDialer, proxyLogger, args.fakeSNI) mainLogger.Info("Init complete.") err = http.ListenAndServe(args.bindAddress, h) } @@ -488,6 +1050,22 @@ func run() int { return 0 } +func printCountryList(list []se.SEGeoEntry) int { + wr := csv.NewWriter(os.Stdout) + defer wr.Flush() + if err := wr.Write([]string{"country code", "country name"}); err != nil { + fmt.Fprintf(os.Stderr, "failed to write country list: %v\n", err) + return 1 + } + for _, country := range list { + if err := wr.Write([]string{country.CountryCode, country.Country}); err != nil { + fmt.Fprintf(os.Stderr, "failed to write country list: %v\n", err) + return 1 + } + } + return 0 +} + func printCountries(try func(string, func() error) error, logger *clog.CondLogger, timeout time.Duration, seclient *se.SEClient) int { var list []se.SEGeoEntry err := try("geolist", func() error { @@ -500,37 +1078,70 @@ func printCountries(try func(string, func() error) error, logger *clog.CondLogge if err != nil { return 11 } - - wr := csv.NewWriter(os.Stdout) - defer wr.Flush() - wr.Write([]string{"country code", "country name"}) - for _, country := range list { - wr.Write([]string{country.CountryCode, country.Country}) - } - return 0 + return printCountryList(list) } -func printProxies(ips []se.SEIPEntry, seclient *se.SEClient) int { - wr := csv.NewWriter(os.Stdout) - defer wr.Flush() +func printProxies(ips []se.SEIPEntry, seclient *se.SEClient, speedResults map[proxyEndpointKey]proxySpeedResult, sortBy string) int { login, password := seclient.GetProxyCredentials() fmt.Println("Proxy login:", login) fmt.Println("Proxy password:", password) fmt.Println("Proxy-Authorization:", dialer.BasicAuthHeader(login, password)) fmt.Println("") - wr.Write([]string{"host", "ip_address", "port"}) - for i, ip := range ips { - for _, port := range ip.Ports { - wr.Write([]string{ - fmt.Sprintf("%s%d.%s", strings.ToLower(ip.Geo.CountryCode), i, PROXY_SUFFIX), - ip.IP, - fmt.Sprintf("%d", port), - }) - } + if err := emitProxyCSV(os.Stdout, ips, speedResults, sortBy); err != nil { + fmt.Fprintf(os.Stderr, "failed to write proxy CSV: %v\n", err) + return 1 } return 0 } +func writeProxyCSV(filename string, ips []se.SEIPEntry, seclient *se.SEClient, speedResults map[proxyEndpointKey]proxySpeedResult, sortBy string) error { + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + return emitProxyCSV(f, ips, speedResults, sortBy) +} + +func emitProxyCSV(w io.Writer, ips []se.SEIPEntry, speedResults map[proxyEndpointKey]proxySpeedResult, sortBy string) error { + wr := csv.NewWriter(w) + defer wr.Flush() + header := []string{"country_code", "country_name", "host", "ip_address", "port"} + includeSpeed := speedResults != nil + if includeSpeed { + header = append(header, "speed_ms", "speed_status") + } + if err := wr.Write(header); err != nil { + return err + } + rows := buildProxyRows(ips, speedResults, sortBy) + for _, rowData := range rows { + row := []string{ + rowData.CountryCode, + rowData.CountryName, + rowData.Host, + rowData.IP, + fmt.Sprintf("%d", rowData.Port), + } + if includeSpeed { + speedMs := "" + status := "not_tested" + if rowData.HasSpeed { + if rowData.Speed.Err == nil { + speedMs = fmt.Sprintf("%d", rowData.Speed.Duration.Milliseconds()) + } + status = rowData.Speed.Status() + } + row = append(row, speedMs, status) + } + if err := wr.Write(row); err != nil { + return err + } + } + wr.Flush() + return wr.Error() +} + func dpExport(ips []se.SEIPEntry, seclient *se.SEClient, sni string) int { wr := csv.NewWriter(os.Stdout) wr.Comma = ' ' @@ -573,6 +1184,634 @@ func sanitizeFixedProxyAddress(addr string) string { return net.JoinHostPort(addr, "443") } +func loadProxyBlacklist(filename string) (map[string]struct{}, error) { + if filename == "" { + return nil, nil + } + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + blacklist := make(map[string]struct{}) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if idx := strings.Index(line, "#"); idx >= 0 { + line = line[:idx] + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + blacklist[sanitizeFixedProxyAddress(line)] = struct{}{} + } + if err := scanner.Err(); err != nil { + return nil, err + } + return blacklist, nil +} + +func filterBlacklistedProxyEntries(entries []se.SEIPEntry, blacklist map[string]struct{}) ([]se.SEIPEntry, []string) { + if len(blacklist) == 0 { + return entries, nil + } + + filtered := make([]se.SEIPEntry, 0, len(entries)) + blocked := make([]string, 0) + blockedSeen := make(map[string]struct{}) + appendBlocked := func(addr string) { + if _, ok := blockedSeen[addr]; ok { + return + } + blockedSeen[addr] = struct{}{} + blocked = append(blocked, addr) + } + + for _, entry := range entries { + if len(entry.Ports) == 0 { + addr := net.JoinHostPort(entry.IP, "443") + if _, ok := blacklist[addr]; ok { + appendBlocked(addr) + continue + } + filtered = append(filtered, entry) + continue + } + + ports := make([]uint16, 0, len(entry.Ports)) + for _, port := range entry.Ports { + addr := net.JoinHostPort(entry.IP, strconv.Itoa(int(port))) + if _, ok := blacklist[addr]; ok { + appendBlocked(addr) + continue + } + ports = append(ports, port) + } + if len(ports) == 0 { + continue + } + filtered = append(filtered, se.SEIPEntry{ + Geo: entry.Geo, + IP: entry.IP, + Ports: ports, + }) + } + + return filtered, blocked +} + +type proxyEndpointKey struct { + countryCode string + ip string + port uint16 +} + +type proxySpeedResult struct { + Duration time.Duration + Err error +} + +type proxyListRow struct { + CountryCode string + CountryName string + Host string + IP string + Port uint16 + Speed proxySpeedResult + HasSpeed bool +} + +func (r proxySpeedResult) Status() string { + if r.Err == nil { + return "ok" + } + return r.Err.Error() +} + +func parseCountryFilters(raw string) ([]string, bool) { + parts := strings.Split(raw, ",") + res := make([]string, 0, len(parts)) + seen := make(map[string]struct{}) + for _, part := range parts { + country := strings.ToUpper(strings.TrimSpace(part)) + if country == "" { + continue + } + if country == "ALL" || country == "*" { + return nil, true + } + if _, ok := seen[country]; ok { + continue + } + seen[country] = struct{}{} + res = append(res, country) + } + return res, false +} + +func proxyEndpointAddrs(entries []se.SEIPEntry) []string { + addrs := make([]string, 0, countProxyPorts(entries)) + for _, entry := range entries { + if len(entry.Ports) == 0 { + addrs = append(addrs, net.JoinHostPort(entry.IP, "443")) + continue + } + for _, port := range entry.Ports { + addrs = append(addrs, net.JoinHostPort(entry.IP, strconv.Itoa(int(port)))) + } + } + return addrs +} + +func discoverCountryWithContext(parent context.Context, args *CLIArgs, seclient *se.SEClient, logger *clog.CondLogger, countryCode string) ([]se.SEIPEntry, error) { + if args.discoverCSV != "" { + return loadProxyEntriesFromCSV(args.discoverCSV, countryCode, false, args.countryExplicit, args.proxyBlacklist) + } + + seen := make(map[proxyEndpointKey]struct{}) + aggregated := make([]se.SEIPEntry, 0) + requestedGeo := fmt.Sprintf("\"%s\",,", countryCode) + for attempt := 1; attempt <= args.discoverRepeat; attempt++ { + ctx, cl := context.WithTimeout(parent, args.timeout) + res, err := seclient.Discover(ctx, requestedGeo) + cl() + if err != nil { + if isSurfEasyDiscover801(err) { + fallbackCSV := discoverCSVPathForFallback(args) + if logger != nil { + logger.Warning("Discover API returned 801 for country %s, falling back to CSV %q.", countryCode, fallbackCSV) + } + res, csvErr := loadProxyEntriesFromCSV(fallbackCSV, countryCode, false, args.countryExplicit, args.proxyBlacklist) + if csvErr != nil { + return nil, fmt.Errorf("discover API returned 801 and CSV fallback %q failed: %w", fallbackCSV, csvErr) + } + return res, nil + } + return nil, err + } + if logger != nil { + logger.Info("Discover for country %s returned %d endpoints on pass #%d: %v", countryCode, len(res), attempt, proxyEndpointAddrs(res)) + } + res, blocked := filterBlacklistedProxyEntries(res, args.proxyBlacklist) + if logger != nil && len(blocked) > 0 { + logger.Info("Discover for country %s skipped %d blacklisted endpoints on pass #%d: %v", countryCode, len(blocked), attempt, blocked) + } + aggregated = appendUniqueProxies(aggregated, res, seen) + } + sortProxyEntries(aggregated) + return aggregated, nil +} + +func discoverCountry(args *CLIArgs, seclient *se.SEClient, logger *clog.CondLogger, countryCode string) ([]se.SEIPEntry, error) { + return discoverCountryWithContext(context.Background(), args, seclient, logger, countryCode) +} + +func discoverAllCountriesWithContext(parent context.Context, args *CLIArgs, seclient *se.SEClient, logger *clog.CondLogger, countryFilter string) ([]se.SEIPEntry, error) { + if args.discoverCSV != "" { + return loadProxyEntriesFromCSV(args.discoverCSV, countryFilter, true, args.countryExplicit, args.proxyBlacklist) + } + + ctx, cl := context.WithTimeout(parent, args.timeout) + countries, err := seclient.GeoList(ctx) + cl() + if err != nil { + return nil, err + } + + filters, allCountries := parseCountryFilters(countryFilter) + if !args.countryExplicit { + allCountries = true + filters = nil + } + allowed := make(map[string]struct{}, len(filters)) + for _, country := range filters { + allowed[country] = struct{}{} + } + + all := make([]se.SEIPEntry, 0) + seen := make(map[proxyEndpointKey]struct{}) + for _, country := range countries { + if !allCountries { + if _, ok := allowed[strings.ToUpper(country.CountryCode)]; !ok { + continue + } + } + res, err := discoverCountryWithContext(parent, args, seclient, logger, country.CountryCode) + if err != nil { + return nil, fmt.Errorf("discover failed for country %s: %w", country.CountryCode, err) + } + all = appendUniqueProxies(all, res, seen) + } + if len(all) == 0 && !allCountries { + return nil, fmt.Errorf("no countries matched filter %q", args.country) + } + sortProxyEntries(all) + return all, nil +} + +func discoverAllCountries(args *CLIArgs, seclient *se.SEClient, logger *clog.CondLogger) ([]se.SEIPEntry, error) { + return discoverAllCountriesWithContext(context.Background(), args, seclient, logger, args.country) +} + +func appendUniqueProxies(dst, src []se.SEIPEntry, seen map[proxyEndpointKey]struct{}) []se.SEIPEntry { + for _, entry := range src { + if len(entry.Ports) == 0 { + key := proxyEndpointKey{ + countryCode: entry.Geo.CountryCode, + ip: entry.IP, + port: 443, + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + dst = append(dst, se.SEIPEntry{ + Geo: entry.Geo, + Host: entry.Host, + IP: entry.IP, + Ports: []uint16{443}, + }) + continue + } + + ports := make([]uint16, 0, len(entry.Ports)) + for _, port := range entry.Ports { + key := proxyEndpointKey{ + countryCode: entry.Geo.CountryCode, + ip: entry.IP, + port: port, + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + ports = append(ports, port) + } + if len(ports) == 0 { + continue + } + dst = append(dst, se.SEIPEntry{ + Geo: entry.Geo, + Host: entry.Host, + IP: entry.IP, + Ports: ports, + }) + } + return dst +} + +func isSurfEasyDiscover801(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "code=801") +} + +func discoverCSVPathForFallback(args *CLIArgs) string { + if args.discoverCSV != "" { + return args.discoverCSV + } + return DefaultDiscoverCSVFallback +} + +func proxyBypassPathForFallback() string { + return DefaultProxyBypassFallback +} + +func loadProxyEntriesFromCSV(filename string, countryFilter string, allowAll bool, countryExplicit bool, blacklist map[string]struct{}) ([]se.SEIPEntry, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + reader := csv.NewReader(f) + reader.FieldsPerRecord = -1 + reader.TrimLeadingSpace = true + + header, err := reader.Read() + if err != nil { + return nil, err + } + indexByName := make(map[string]int, len(header)) + for i, name := range header { + indexByName[strings.ToLower(strings.TrimSpace(name))] = i + } + + requiredColumns := []string{"country_code", "ip_address", "port"} + for _, col := range requiredColumns { + if _, ok := indexByName[col]; !ok { + return nil, fmt.Errorf("proxy CSV %q is missing required column %q", filename, col) + } + } + + filters, allCountries := parseCountryFilters(countryFilter) + if allowAll && !countryExplicit { + allCountries = true + filters = nil + } + allowedCountries := make(map[string]struct{}, len(filters)) + for _, country := range filters { + allowedCountries[country] = struct{}{} + } + + seen := make(map[proxyEndpointKey]struct{}) + entries := make([]se.SEIPEntry, 0) + lineNo := 1 + fieldValue := func(record []string, column string) (string, error) { + idx := indexByName[column] + if idx >= len(record) { + return "", fmt.Errorf("proxy CSV %q line %d is missing value for %s", filename, lineNo, column) + } + return strings.TrimSpace(record[idx]), nil + } + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to read proxy CSV %q at line %d: %w", filename, lineNo+1, err) + } + lineNo++ + + countryCodeValue, err := fieldValue(record, "country_code") + if err != nil { + return nil, err + } + countryCode := strings.ToUpper(countryCodeValue) + if countryCode == "" { + return nil, fmt.Errorf("proxy CSV %q line %d has empty country_code", filename, lineNo) + } + if !allCountries { + if _, ok := allowedCountries[countryCode]; !ok { + continue + } + } + + ipAddr, err := fieldValue(record, "ip_address") + if err != nil { + return nil, err + } + if ipAddr == "" { + return nil, fmt.Errorf("proxy CSV %q line %d has empty ip_address", filename, lineNo) + } + if net.ParseIP(ipAddr) == nil { + return nil, fmt.Errorf("proxy CSV %q line %d has invalid ip_address %q", filename, lineNo, ipAddr) + } + + portValue, err := fieldValue(record, "port") + if err != nil { + return nil, err + } + portNum, err := strconv.ParseUint(portValue, 10, 16) + if err != nil { + return nil, fmt.Errorf("proxy CSV %q line %d has invalid port %q: %w", filename, lineNo, portValue, err) + } + + countryName := "" + if idx, ok := indexByName["country_name"]; ok && idx < len(record) { + countryName = strings.TrimSpace(record[idx]) + } + hostName := "" + if idx, ok := indexByName["host"]; ok && idx < len(record) { + hostName = strings.TrimSpace(record[idx]) + } + + entry := se.SEIPEntry{ + Geo: se.SEGeoEntry{ + CountryCode: countryCode, + Country: countryName, + }, + Host: hostName, + IP: ipAddr, + Ports: []uint16{uint16(portNum)}, + } + entries = appendUniqueProxies(entries, []se.SEIPEntry{entry}, seen) + } + + entries, _ = filterBlacklistedProxyEntries(entries, blacklist) + if len(entries) == 0 { + if allowAll { + if allCountries { + return nil, fmt.Errorf("no proxy endpoints found in CSV %q", filename) + } + return nil, fmt.Errorf("no proxy endpoints from CSV %q matched country filter %q", filename, countryFilter) + } + return nil, fmt.Errorf("no proxy endpoints from CSV %q matched country %q", filename, countryFilter) + } + + sortProxyEntries(entries) + return entries, nil +} + +func sortProxyEntries(entries []se.SEIPEntry) { + for i := range entries { + sort.Slice(entries[i].Ports, func(a, b int) bool { + return entries[i].Ports[a] < entries[i].Ports[b] + }) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Geo.CountryCode != entries[j].Geo.CountryCode { + return entries[i].Geo.CountryCode < entries[j].Geo.CountryCode + } + if entries[i].IP != entries[j].IP { + return entries[i].IP < entries[j].IP + } + leftPort, rightPort := uint16(443), uint16(443) + if len(entries[i].Ports) > 0 { + leftPort = entries[i].Ports[0] + } + if len(entries[j].Ports) > 0 { + rightPort = entries[j].Ports[0] + } + return leftPort < rightPort + }) +} + +func countProxyPorts(entries []se.SEIPEntry) int { + total := 0 + for _, entry := range entries { + if len(entry.Ports) == 0 { + total++ + continue + } + total += len(entry.Ports) + } + return total +} + +func buildProxyRows(ips []se.SEIPEntry, speedResults map[proxyEndpointKey]proxySpeedResult, sortBy string) []proxyListRow { + rows := make([]proxyListRow, 0, countProxyPorts(ips)) + for i, ip := range ips { + ports := ip.Ports + if len(ports) == 0 { + ports = []uint16{443} + } + for _, port := range ports { + row := proxyListRow{ + CountryCode: ip.Geo.CountryCode, + CountryName: ip.Geo.Country, + Host: ip.Host, + IP: ip.IP, + Port: port, + } + if row.Host == "" { + row.Host = fmt.Sprintf("%s%d.%s", strings.ToLower(ip.Geo.CountryCode), i, PROXY_SUFFIX) + } + if speedResults != nil { + result, ok := speedResults[proxyEndpointKey{ + countryCode: ip.Geo.CountryCode, + ip: ip.IP, + port: port, + }] + row.HasSpeed = ok + if ok { + row.Speed = result + } + } + rows = append(rows, row) + } + } + + sortProxyRows(rows, sortBy) + + return rows +} + +func sortProxyRows(rows []proxyListRow, sortBy string) { + sort.SliceStable(rows, func(i, j int) bool { + left, right := rows[i], rows[j] + switch sortBy { + case "country": + if left.CountryCode != right.CountryCode { + return left.CountryCode < right.CountryCode + } + if left.CountryName != right.CountryName { + return left.CountryName < right.CountryName + } + if left.IP != right.IP { + return left.IP < right.IP + } + return left.Port < right.Port + case "ip": + if left.IP != right.IP { + return left.IP < right.IP + } + if left.Port != right.Port { + return left.Port < right.Port + } + return left.CountryCode < right.CountryCode + default: + leftOK := left.HasSpeed && left.Speed.Err == nil + rightOK := right.HasSpeed && right.Speed.Err == nil + if leftOK != rightOK { + return leftOK + } + if leftOK && rightOK && left.Speed.Duration != right.Speed.Duration { + return left.Speed.Duration < right.Speed.Duration + } + if left.CountryCode != right.CountryCode { + return left.CountryCode < right.CountryCode + } + if left.IP != right.IP { + return left.IP < right.IP + } + return left.Port < right.Port + } + }) +} + +func benchmarkProxyEndpoints(args *CLIArgs, ips []se.SEIPEntry, caPool *x509.CertPool, logger *clog.CondLogger, dialerFactory func(se.SEIPEntry, string) dialer.ContextDialer) map[proxyEndpointKey]proxySpeedResult { + results := make(map[proxyEndpointKey]proxySpeedResult) + var mu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, 8) + + for _, entry := range ips { + ports := entry.Ports + if len(ports) == 0 { + ports = []uint16{443} + } + for _, port := range ports { + key := proxyEndpointKey{ + countryCode: entry.Geo.CountryCode, + ip: entry.IP, + port: port, + } + endpoint := net.JoinHostPort(entry.IP, strconv.Itoa(int(port))) + wg.Add(1) + go func(entry se.SEIPEntry, key proxyEndpointKey, countryCode, endpoint string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + start := time.Now() + ctx, cl := context.WithTimeout(context.Background(), args.proxySpeedTimeout) + err := probeProxyEndpoint(ctx, dialerFactory(entry, endpoint), args.proxySpeedTestURL, args.proxySpeedDLLimit, &tls.Config{ + RootCAs: caPool, + }) + cl() + + result := proxySpeedResult{ + Duration: time.Since(start), + Err: err, + } + + mu.Lock() + results[key] = result + mu.Unlock() + + if err == nil { + logger.Info("Speed probe for %s via %s completed in %d ms.", countryCode, endpoint, result.Duration.Milliseconds()) + } else { + logger.Warning("Speed probe for %s via %s failed: %v", countryCode, endpoint, err) + } + }(entry, key, entry.Geo.CountryCode, endpoint) + } + } + + wg.Wait() + return results +} + +func probeProxyEndpoint(ctx context.Context, upstream dialer.ContextDialer, targetURL string, dlLimit int64, tlsClientConfig *tls.Config) error { + httpClient := http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: upstream.DialContext, + TLSClientConfig: tlsClientConfig, + ForceAttemptHTTP2: true, + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if err != nil { + return err + } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http_%d", resp.StatusCode) + } + + var reader io.Reader = resp.Body + if dlLimit > 0 { + reader = io.LimitReader(reader, dlLimit) + } + _, err = io.Copy(io.Discard, reader) + return err +} + func main() { os.Exit(run()) } diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..1b341df --- /dev/null +++ b/main_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLoadProxyBypassList(t *testing.T) { + dir := t.TempDir() + filename := filepath.Join(dir, "proxy-bypass.txt") + content := "" + + "# comment only\n" + + "api2.sec-tunnel.com\n" + + "*.example.com, https://download.test.local/list.txt\n" + + "api2.sec-tunnel.com # duplicate\n" + + "\n" + if err := os.WriteFile(filename, []byte(content), 0o644); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + + got, err := loadProxyBypassList(filename) + if err != nil { + t.Fatalf("loadProxyBypassList() error = %v", err) + } + + want := []string{ + "api2.sec-tunnel.com", + "*.example.com", + "https://download.test.local/list.txt", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("loadProxyBypassList() = %#v, want %#v", got, want) + } +} diff --git a/proxies.csv b/proxies.csv new file mode 100644 index 0000000..cc87b3b --- /dev/null +++ b/proxies.csv @@ -0,0 +1,21 @@ +country_code,country_name,host,ip_address,port,speed_ms,speed_status +EU,,eu14.sec-tunnel.com,77.111.247.43,443,566,ok +EU,,eu15.sec-tunnel.com,77.111.247.44,443,577,ok +EU,,eu17.sec-tunnel.com,77.111.247.55,443,587,ok +EU,,eu13.sec-tunnel.com,77.111.247.233,443,600,ok +EU,,eu16.sec-tunnel.com,77.111.247.46,443,606,ok +EU,,eu19.sec-tunnel.com,77.111.247.57,443,607,ok +EU,,eu12.sec-tunnel.com,77.111.247.232,443,622,ok +EU,,eu18.sec-tunnel.com,77.111.247.56,443,651,ok +AM,,am1.sec-tunnel.com,77.111.246.14,443,1269,ok +AM,,am2.sec-tunnel.com,77.111.246.60,443,1269,ok +AM,,am0.sec-tunnel.com,77.111.246.126,443,1427,ok +EU,,eu11.sec-tunnel.com,77.111.244.37,443,1429,ok +EU,,eu10.sec-tunnel.com,77.111.244.210,443,1535,ok +AS,,as8.sec-tunnel.com,77.111.245.16,443,2200,ok +AS,,as5.sec-tunnel.com,77.111.245.10,443,2263,ok +AS,,as7.sec-tunnel.com,77.111.245.15,443,2359,ok +AS,,as6.sec-tunnel.com,77.111.245.12,443,2430,ok +AS,,as9.sec-tunnel.com,77.111.245.17,443,3300,ok +AM,,am3.sec-tunnel.com,77.111.246.62,443,11270,ok +AM,,am4.sec-tunnel.com,77.111.246.64,443,11654,ok diff --git a/proxies.txt b/proxies.txt new file mode 100644 index 0000000..4de950b --- /dev/null +++ b/proxies.txt @@ -0,0 +1,436 @@ +106.51.185.233:8080 +213.169.33.10:8001 +103.227.187.1:6080 +131.161.68.41:35944 +61.49.87.3:80 +203.28.67.74:8080 +152.42.201.82:1080 +82.115.60.66:80 +24.152.58.108:999 +190.60.47.62:999 +176.88.168.107:8080 +38.51.207.184:999 +103.136.171.145:8080 +118.69.186.75:1452 +200.35.34.134:999 +204.157.251.178:999 +185.76.240.94:10001 +176.88.166.207:8080 +51.159.28.39:80 +112.203.192.108:8082 +2.27.32.81:3128 +79.137.204.101:1080 +46.107.230.122:1080 +8.209.239.31:30000 +179.49.113.225:999 +193.43.159.163:8080 +186.248.87.172:5678 +138.68.235.51:80 +118.70.151.55:1080 +79.101.106.74:33608 +58.33.109.114:2021 +103.78.83.22:8083 +185.76.240.97:10001 +192.252.211.197:14921 +168.138.202.218:3128 +77.91.77.220:3128 +103.132.52.229:8080 +185.76.240.99:10001 +138.84.65.117:8081 +187.111.144.102:8080 +181.94.197.37:8080 +185.242.107.91:3129 +5.102.109.41:999 +185.76.240.91:10001 +31.97.100.157:8080 +114.111.151.41:80 +5.196.101.18:3128 +102.68.135.147:8080 +194.87.215.166:9443 +14.232.166.150:5678 +185.114.73.2:1080 +8.212.177.126:8080 +103.153.246.54:8181 +103.46.186.161:8080 +201.234.50.194:3128 +210.223.44.230:3128 +176.61.151.123:80 +213.131.85.28:1976 +45.92.108.112:80 +135.148.120.6:80 +108.161.135.118:80 +138.124.99.216:8888 +103.103.146.149:7080 +79.137.17.104:80 +185.85.111.18:80 +112.198.22.122:80 +80.78.73.142:65530 +94.23.9.170:80 +104.225.220.233:80 +133.18.234.13:80 +34.135.166.24:80 +195.231.69.203:80 +193.105.62.11:58973 +186.26.95.249:61445 +174.104.115.21:80 +35.209.198.222:80 +167.99.124.118:80 +201.222.50.218:80 +174.138.119.88:80 +185.76.241.223:10001 +159.223.225.118:8888 +163.172.167.48:80 +65.108.103.19:80 +46.17.47.48:80 +13.80.134.180:80 +154.43.62.22:80 +190.119.132.61:80 +46.21.186.252:1080 +147.231.163.133:80 +124.108.6.20:8085 +5.161.103.41:88 +143.198.135.176:80 +103.205.64.153:80 +190.110.226.122:80 +37.187.74.125:80 +185.76.241.158:10001 +103.151.20.131:80 +66.111.113.34:80 +183.110.216.159:8090 +31.220.78.244:80 +95.214.9.93:3128 +23.247.136.254:80 +185.76.240.64:10001 +194.102.38.53:80 +62.149.165.171:80 +185.76.240.60:10001 +103.65.237.92:5678 +138.91.159.185:80 +219.93.101.62:80 +185.76.240.61:10001 +154.65.39.7:80 +79.174.12.190:80 +197.221.240.240:80 +51.79.135.131:8080 +152.53.65.215:8090 +200.174.198.32:8888 +167.99.236.14:80 +27.34.242.98:80 +141.147.9.254:80 +13.230.49.39:8080 +149.248.215.39:8081 +113.160.132.26:8080 +8.219.97.248:80 +209.135.168.41:80 +203.23.238.21:80 +41.220.16.213:80 +163.172.53.142:80 +34.44.49.215:80 +152.230.215.123:80 +211.38.188.120:9080 +35.225.22.61:80 +45.146.163.31:80 +213.177.2.55:80 +41.220.16.209:80 +196.1.93.16:80 +197.221.249.196:80 +195.26.224.135:80 +219.249.37.107:8382 +168.222.254.88:3128 +156.38.112.11:80 +212.231.191.23:80 +170.64.170.204:8080 +45.167.125.21:999 +102.223.9.53:80 +69.70.244.34:80 +97.74.87.226:80 +190.119.132.62:80 +219.65.73.81:80 +210.177.178.148:80 +87.255.196.143:80 +41.220.16.218:80 +219.93.101.63:80 +219.93.101.60:80 +85.214.204.79:80 +39.109.113.97:4090 +196.223.129.21:80 +34.81.160.132:80 +38.60.196.214:80 +194.71.227.149:80 +197.221.240.178:80 +78.28.152.113:80 +41.220.16.208:80 +154.65.39.8:80 +197.221.234.252:80 +197.221.249.198:80 +41.220.16.215:80 +193.181.35.106:8118 +46.249.100.124:80 +194.150.110.134:80 +147.91.22.150:80 +159.65.221.25:80 +139.162.200.213:80 +150.136.163.51:80 +91.132.92.231:80 +93.127.163.52:80 +91.132.92.150:80 +34.140.137.151:80 +198.111.166.184:80 +192.73.244.36:80 +66.135.16.53:80 +89.58.55.33:80 +151.236.24.38:80 +207.180.254.198:8080 +212.47.232.28:80 +84.39.112.144:3128 +202.133.88.173:80 +80.74.54.148:3128 +109.236.88.82:80 +178.156.224.42:3128 +195.91.129.101:1337 +103.90.66.19:1080 +187.19.127.246:8011 +138.36.199.14:4153 +41.223.234.116:37259 +110.93.206.62:5678 +194.61.24.198:1080 +103.37.82.134:39873 +103.136.107.70:1080 +14.241.241.185:4145 +110.232.86.221:1080 +83.218.186.22:5678 +103.174.122.197:8199 +110.235.248.142:1080 +69.48.201.94:80 +31.43.194.184:1080 +119.148.47.166:22122 +76.81.6.107:31008 +103.88.169.106:33149 +195.138.65.34:5678 +109.166.207.162:3629 +91.247.92.63:5678 +81.228.44.187:65530 +181.115.75.102:5678 +113.53.29.228:13629 +103.111.136.82:8199 +168.253.92.93:10808 +138.68.247.51:1080 +36.37.244.41:5678 +103.156.14.234:1080 +160.22.22.85:5678 +45.236.152.21:4145 +177.136.124.47:56113 +111.67.103.162:1080 +128.199.37.92:1080 +202.70.80.153:5678 +192.248.95.98:54126 +34.122.187.196:80 +186.47.213.158:5678 +46.146.216.44:1080 +87.117.11.57:1080 +20.64.237.168:1080 +202.51.103.154:5678 +198.0.198.132:54321 +163.47.37.190:1080 +125.24.156.113:7080 +139.255.63.170:14888 +46.30.46.133:3128 +103.39.245.106:1080 +202.84.76.190:5678 +193.233.254.82:1080 +206.189.154.204:8888 +31.43.63.70:4145 +199.229.254.129:4145 +187.157.30.202:4153 +114.130.153.122:58080 +192.252.214.20:15864 +144.124.227.145:8080 +203.189.135.73:1080 +213.169.33.8:8001 +202.5.53.145:9355 +103.172.120.102:8097 +198.8.94.174:39078 +202.40.188.2:1088 +192.111.135.17:18302 +192.252.209.158:4145 +124.83.114.43:8081 +41.21.182.179:5678 +103.175.127.230:60606 +103.102.12.134:1111 +103.19.58.151:8080 +45.79.203.254:48388 +143.137.116.72:1080 +103.120.221.129:8083 +202.154.241.199:808 +103.99.110.222:5678 +103.151.74.5:2025 +5.134.48.59:8080 +202.154.19.63:8083 +103.17.246.60:1080 +103.156.248.78:8080 +202.74.243.182:228 +97.105.12.186:4153 +103.163.244.106:1080 +102.212.44.151:12354 +163.53.204.178:9813 +157.15.40.251:7777 +79.132.136.58:3128 +50.238.47.86:32100 +192.111.135.18:18301 +103.132.52.32:8080 +47.237.169.183:1080 +202.181.16.45:52929 +192.252.209.155:14455 +94.247.241.70:51006 +65.21.201.149:8080 +147.45.167.84:3128 +193.233.254.61:1080 +5.255.123.43:1080 +192.111.139.165:4145 +20.64.246.197:1080 +114.130.175.146:1088 +194.67.99.223:1080 +91.108.243.203:3128 +91.150.189.122:60647 +192.252.210.233:4145 +157.66.16.52:8080 +91.99.182.39:4000 +31.43.33.56:4153 +8.220.143.77:1080 +45.148.30.212:3128 +89.43.133.146:8080 +168.110.52.228:3128 +103.56.115.156:7890 +181.236.221.138:4145 +103.167.156.202:1080 +103.183.19.34:3128 +41.139.147.86:5678 +113.192.31.17:8080 +203.189.141.138:1080 +103.97.140.157:1080 +37.60.237.110:30001 +65.108.203.36:18080 +192.252.216.81:4145 +192.111.129.150:4145 +103.153.63.223:1080 +103.189.63.149:53053 +198.8.94.170:4145 +38.76.200.182:58367 +162.240.19.30:80 +185.76.240.178:10001 +41.169.151.154:4153 +103.180.123.27:8080 +5.161.50.82:8118 +121.135.144.141:8096 +75.84.71.14:80 +193.23.194.147:3128 +185.76.241.162:10001 +196.1.97.198:80 +185.76.240.208:10001 +185.76.240.45:10001 +46.47.197.210:3128 +23.88.88.102:80 +45.12.151.226:2829 +196.1.93.10:80 +185.76.240.203:10001 +150.107.140.238:3128 +41.184.92.220:80 +8.209.255.13:3128 +143.42.66.91:80 +46.29.162.166:80 +41.184.92.219:80 +23.88.88.105:443 +116.203.139.209:5678 +176.236.37.132:1080 +8.211.152.121:10000 +20.57.134.61:1080 +202.62.34.102:1080 +195.133.250.173:3128 +110.235.247.105:1080 +103.112.234.65:5678 +212.23.69.121:1080 +109.120.151.2:1080 +203.189.154.80:1080 +213.149.154.213:5678 +117.102.115.154:4153 +202.4.117.136:12321 +103.239.52.100:1080 +210.16.86.105:5678 +168.232.213.9:4153 +20.120.246.129:1080 +31.13.239.4:5678 +31.10.83.158:8080 +27.123.254.158:6969 +23.133.196.12:9000 +202.166.196.105:48293 +102.36.127.53:1080 +154.79.248.156:5678 +95.179.255.75:1080 +119.148.51.30:22122 +191.7.208.86:8080 +122.252.179.66:5678 +171.249.163.170:1452 +49.151.177.202:8082 +107.219.228.250:7777 +113.160.188.21:1080 +103.18.47.79:4145 +223.25.110.37:8199 +190.60.40.242:999 +81.12.157.98:5678 +115.127.112.178:1080 +103.10.99.110:5678 +124.105.55.176:30906 +130.193.123.34:5678 +160.19.41.60:80 +31.42.6.125:5678 +190.102.233.100:999 +158.160.215.167:8123 +103.206.68.241:1080 +27.147.153.34:10888 +123.200.8.6:1081 +113.161.6.170:1080 +185.14.149.33:4145 +103.239.201.50:58765 +165.0.100.149:5678 +118.174.233.113:4153 +110.74.195.152:1080 +164.138.205.232:8080 +80.78.74.8:65530 +185.95.186.165:48293 +202.138.240.249:8080 +45.81.146.7:8080 +163.61.254.104:1111 +103.39.51.184:8088 +107.172.102.234:40621 +197.164.101.13:1981 +181.48.243.194:4153 +82.209.251.53:45678 +1.179.172.45:31225 +209.14.108.98:999 +120.28.193.126:8082 +38.58.170.10:999 +103.66.62.177:8080 +205.177.85.130:39593 +103.30.211.34:80 +91.211.115.127:1080 +202.58.77.194:8031 +37.26.86.206:4145 +62.4.37.104:60606 +91.147.235.162:4153 +185.76.240.33:10001 +43.153.28.68:3128 +185.76.240.57:10001 +185.76.240.12:10001 +185.76.241.132:10001 +185.76.241.232:10001 +185.76.240.19:10001 +167.71.196.28:8080 +185.76.240.177:10001 +185.76.240.18:10001 +185.76.240.47:10001 +172.236.140.124:3128 +185.76.240.55:10001 +185.76.240.201:10001 +91.121.63.51:1080 +62.113.119.14:8080 +213.154.2.210:3128 diff --git a/proxy-bypass.txt b/proxy-bypass.txt new file mode 100644 index 0000000..612d98d --- /dev/null +++ b/proxy-bypass.txt @@ -0,0 +1,19 @@ +# комментарий +*.ru + +#avito +avito.ru +*.avito.ru +avito.st +cdn.avito.ru +img.avito.ru +static.avito.ru + +#ozon +ozon.ru +ozon.com +*.ozon.ru +ozonusercontent.com +ozon-images.ru +ozon-st.com +cdn.ozon.ru \ No newline at end of file diff --git a/seclient/messages.go b/seclient/messages.go index b4d9f9f..14c6dc3 100644 --- a/seclient/messages.go +++ b/seclient/messages.go @@ -12,6 +12,22 @@ const ( SE_STATUS_OK int64 = 0 ) +type APIError struct { + Code int64 + Message string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API responded with error message: code=%d, msg=%q", e.Code, e.Message) +} + +func newAPIError(status SEStatusPair) error { + return &APIError{ + Code: status.Code, + Message: status.Message, + } +} + type SEStatusPair struct { Code int64 Message string @@ -85,6 +101,7 @@ type SEGeoListResponse struct { type SEIPEntry struct { Geo SEGeoEntry `json:"geo"` + Host string `json:"host,omitempty"` IP string `json:"ip"` Ports []uint16 `json:"ports"` } diff --git a/seclient/seclient.go b/seclient/seclient.go index fb1c7d5..d1490a0 100644 --- a/seclient/seclient.go +++ b/seclient/seclient.go @@ -153,8 +153,7 @@ func (c *SEClient) register(ctx context.Context) error { } if regRes.Status.Code != SE_STATUS_OK { - return fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - regRes.Status.Code, regRes.Status.Message) + return newAPIError(regRes.Status) } return nil } @@ -174,8 +173,7 @@ func (c *SEClient) RegisterDevice(ctx context.Context) error { } if regRes.Status.Code != SE_STATUS_OK { - return fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - regRes.Status.Code, regRes.Status.Message) + return newAPIError(regRes.Status) } c.AssignedDeviceID = regRes.Data.DeviceID @@ -197,8 +195,7 @@ func (c *SEClient) GeoList(ctx context.Context) ([]SEGeoEntry, error) { } if geoListRes.Status.Code != SE_STATUS_OK { - return nil, fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - geoListRes.Status.Code, geoListRes.Status.Message) + return nil, newAPIError(geoListRes.Status) } return geoListRes.Data.Geos, nil @@ -218,8 +215,7 @@ func (c *SEClient) Discover(ctx context.Context, requestedGeo string) ([]SEIPEnt } if discoverRes.Status.Code != SE_STATUS_OK { - return nil, fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - discoverRes.Status.Code, discoverRes.Status.Message) + return nil, newAPIError(discoverRes.Status) } return discoverRes.Data.IPs, nil @@ -245,8 +241,7 @@ func (c *SEClient) Login(ctx context.Context) error { } if loginRes.Status.Code != SE_STATUS_OK { - return fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - loginRes.Status.Code, loginRes.Status.Message) + return newAPIError(loginRes.Status) } return nil } @@ -264,8 +259,7 @@ func (c *SEClient) DeviceGeneratePassword(ctx context.Context) error { } if genRes.Status.Code != SE_STATUS_OK { - return fmt.Errorf("API responded with error message: code=%d, msg=\"%s\"", - genRes.Status.Code, genRes.Status.Message) + return newAPIError(genRes.Status) } c.DevicePassword = genRes.Data.DevicePassword