ruantiblock-mod-lua, ruantiblock-mod-py: subdomains optimization.

autoinstall/2.x/autoinstall.sh

@@ -10,9 +10,9 @@ LUCI_APP=1
 HTTPS_DNS_PROXY=1
 
 OWRT_VERSION="current"
-RUAB_VERSION="2.1.2-r1"
+RUAB_VERSION="2.1.3-r1"
-RUAB_MOD_LUA_VERSION="2.1.2-r1"
+RUAB_MOD_LUA_VERSION="2.1.3-r1"
-RUAB_LUCI_APP_VERSION="2.1.2-r1"
+RUAB_LUCI_APP_VERSION="2.1.3-r1"
 BASE_URL="https://raw.githubusercontent.com/gSpotx2f/packages-openwrt/master"
 PKG_DIR="/tmp"
 

luci-app-ruantiblock/Makefile

@@ -5,7 +5,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=luci-app-ruantiblock
-PKG_VERSION:=2.1.2
+PKG_VERSION:=2.1.3
 PKG_RELEASE:=1
 LUCI_TITLE:=LuCI support for ruantiblock
 LUCI_DEPENDS:=+ruantiblock

luci-app-ruantiblock/htdocs/luci-static/resources/view/ruantiblock/settings.js

@@ -149,13 +149,6 @@ return view.extend({
 				'<br /><code>poker<br />[ck]?a[sz]ino?<br />[vw]ulkan<br />slots?</code>'
 		);
 
-		let user_entries_edit = new tools.fileEditDialog(
-			tools.userEntriesFile,
-			_('User entries'),
-			_('One entry (IP, CIDR or FQDN) per line. In the FQDN records, you can specify the DNS server for resolving this domain (separated by a space). You can also comment on lines (<code>#</code> is the first character of a line).<br />Examples:') +
-				'<br /><code>#comment<br />domain.net<br />sub.domain.com 8.8.8.8<br />sub.domain.com 8.8.8.8#53<br />74.125.131.19<br />74.125.0.0/16</code>'
-		);
-
 		let bypass_entries_edit = new tools.fileEditDialog(
 			tools.bypassEntriesFile,
 			_('Exclusion list'),

ruantiblock-mod-lua/Makefile

@@ -5,7 +5,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock-mod-lua
-PKG_VERSION:=2.1.2
+PKG_VERSION:=2.1.3
 PKG_RELEASE:=1
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 
@@ -18,6 +18,7 @@ define Package/$(PKG_NAME)
   URL:=https://github.com/gSpotx2f/ruantiblock_openwrt
   PKGARCH:=all
   DEPENDS:=+ruantiblock +lua +luasocket +luasec +luabitop +iconv +idn
+  #DEPENDS:=+ruantiblock +lua +luasocket +luasec +luabitop +iconv +idn +lua-lzlib
 endef
 
 define Package/$(PKG_NAME)/description

ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua

@@ -251,6 +251,12 @@ if not it then
     Config.BLLIST_SUMMARIZE_CIDR = false
     Config.BLLIST_SUMMARIZE_IP = false
 end
+--[[
+local zlib = prequire("zlib")
+if Config.BLLIST_ENABLE_IDN and not idn then
+    error("You need to install lua-lzlib...")
+end
+--]]
 
 -- Iconv check
 
@@ -392,6 +398,12 @@ function BlackListParser:parser_func()
     error("Method BlackListParser:parser_func() must be overridden by a subclass!")
 end
 
+function BlackListParser:prepare_data()
+    return function(chunk)
+        return chunk
+    end
+end
+
 function BlackListParser:chunk_buffer()
     local ret_value = ""
     local last_chunk
@@ -425,7 +437,7 @@ function BlackListParser:get_http_data(url)
         end
         ret_val, ret_code, ret_headers = http_module.request{url = url, method="HEAD", headers = self.http_send_headers}
         if ret_val and ret_code == 200 then
-            local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
+            local http_sink = ltn12.sink.chain(self:prepare_data(), self:chunk_buffer(), self:parser_func())
             ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers}
             self.http_codes[ret_code] = true
             if not ret_val or ret_code ~= 200 then
@@ -437,7 +449,7 @@ function BlackListParser:get_http_data(url)
             print(string.format("Connection error! (%s) URL: %s", ret_code, url))
         end
     else
-        local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
+        local wget_sink = ltn12.sink.chain(self:prepare_data(), self:chunk_buffer(), self:parser_func())
         ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink)
     end
     return (ret_val == 1) and true or false
@@ -695,21 +707,25 @@ function OptimizeConfig:new(t)
     return instance
 end
 
-function OptimizeConfig:_optimize_ip_table()
+function OptimizeConfig:_remove_subdomains()
-    local optimized_table = {}
+    local tld_table = {}
-    for ipaddr, subnet in pairs(self.ip_table) do
+    for fqdn, sld in pairs(self.fqdn_table) do
-        if self.ip_subnet_table[subnet] then
+        if not tld_table[sld] then
-            if (self.BLLIST_IP_LIMIT and self.BLLIST_IP_LIMIT > 0 and not self.BLLIST_GR_EXCLUDED_NETS_PATTERNS[subnet]) and self.ip_subnet_table[subnet] >= self.BLLIST_IP_LIMIT then
+            tld_table[sld] = {}
-                self.cidr_table[string.format("%s0/24", subnet)] = true
+        end
-                self.ip_subnet_table[subnet] = nil
+        tld_table[sld][fqdn] = true
-                self.cidr_count = self.cidr_count + 1
+    end
-            else
+    for _, v in pairs(tld_table) do
-                optimized_table[ipaddr] = true
+        for i in pairs(v) do
-                self.ip_records_count = self.ip_records_count + 1
+            if self.fqdn_table[i] then
+                for j in pairs(v) do
+                    if (j ~= i) and j:find("." .. i, 1, true) then
+                        self.fqdn_table[j] = nil
+                    end
+                end
             end
         end
     end
-    self.ip_table = optimized_table
 end
 
 function OptimizeConfig:_optimize_fqdn_table()
@@ -728,6 +744,23 @@ function OptimizeConfig:_optimize_fqdn_table()
     self.fqdn_table = optimized_table
 end
 
+function OptimizeConfig:_optimize_ip_table()
+    local optimized_table = {}
+    for ipaddr, subnet in pairs(self.ip_table) do
+        if self.ip_subnet_table[subnet] then
+            if (self.BLLIST_IP_LIMIT and self.BLLIST_IP_LIMIT > 0 and not self.BLLIST_GR_EXCLUDED_NETS_PATTERNS[subnet]) and self.ip_subnet_table[subnet] >= self.BLLIST_IP_LIMIT then
+                self.cidr_table[string.format("%s0/24", subnet)] = true
+                self.ip_subnet_table[subnet] = nil
+                self.cidr_count = self.cidr_count + 1
+            else
+                optimized_table[ipaddr] = true
+                self.ip_records_count = self.ip_records_count + 1
+            end
+        end
+    end
+    self.ip_table = optimized_table
+end
+
 function OptimizeConfig:_group_ip_ranges()
     for i in Summarize:summarize_ip_ranges(self.ip_table, true) do
         self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true
@@ -759,6 +792,7 @@ function OptimizeConfig:optimize()
         self:_union(self.fqdn_table, i.fqdn_table)
         self:_union(self.sld_table, i.sld_table)
     end
+    self:_remove_subdomains()
     self:_optimize_fqdn_table()
     self:_optimize_ip_table()
     if self.BLLIST_SUMMARIZE_IP then
@@ -909,6 +943,43 @@ local Zi = Class(BlackListParser, {
     site_encoding = Config.ZI_ENCODING,
 })
 
+-- for https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
+-- lua-lzlib --
+--[[
+function Zi:prepare_data()
+    local curr_chunk
+    local stream = zlib.inflate({
+        read = function(self, consume)
+            return curr_chunk
+        end,
+    })
+    return function(chunk)
+        if chunk then
+            curr_chunk = chunk
+            return (stream:read(2048)) or ""
+        end
+        stream:close()
+        return nil
+    end
+end
+--]]
+-- lua-zlib --
+--[[
+function Zi:prepare_data()
+    local stream = zlib.inflate()
+    return function(chunk)
+        if chunk then
+            local inflated, eos = stream(chunk)
+            if eos then
+                stream = zlib.inflate()
+            end
+            return inflated
+        end
+        return nil
+    end
+end
+--]]
+
 function Zi:parser_func()
     return function(chunk)
         if chunk and chunk ~= "" then
@@ -1075,7 +1146,6 @@ if parser_classes then
     for _, i in ipairs(parser_instances) do
         ret_list[i:run()] = true
     end
-
     local return_sum = 0
     for i, _ in pairs(ret_list) do
         return_sum = return_sum + i
@@ -1091,5 +1161,4 @@ if parser_classes then
 else
     error("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)")
 end
-
 os.exit(ret_list[1] and 1 or (ret_list[2] and 2 or 0))

ruantiblock-mod-py/Makefile

@@ -5,7 +5,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock-mod-py
-PKG_VERSION:=2.1.2
+PKG_VERSION:=2.1.3
 PKG_RELEASE:=1
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 

ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py

@@ -15,6 +15,7 @@ import socket
 import ssl
 import sys
 from urllib import request
+#import zlib
 
 
 class Config:
@@ -281,8 +282,12 @@ class BlackListParser(Config):
                     if not chunk:
                         break
 
-    def _align_chunk(self, url):
+    def prepare_data(self, url):
         for chunk in self._download_data(url):
+            yield chunk
+
+    def _align_chunk(self, url):
+        for chunk in self.prepare_data(url):
             if chunk is None:
                 yield self.rest
                 continue
@@ -458,7 +463,6 @@ class Summarize:
 
     @classmethod
     def _group_nets(cls, cidr_list, raw_list=None):
-
         def remove_items(start, end):
             for ip in range(int(start), int(end) + 1, 256):
                 raw_list.remove(str(IPv4Address(ip)) + "/24")
@@ -507,6 +511,18 @@ class OptimizeConfig(Config):
         self.ip_count = 0
         self.output_fqdn_count = 0
 
+    def _remove_subdomains(self):
+        tld_dict = {}
+        for fqdn, sld in self.fqdn_dict.items():
+            tld_dict.setdefault(sld, [])
+            tld_dict[sld].append(fqdn)
+        for v in tld_dict.values():
+            for i in v:
+                if i in self.fqdn_dict:
+                    for j in v:
+                        if (j != i) and j.endswith("." + i):
+                            self.fqdn_dict.pop(j, None)
+
     def _optimize_fqdn_dict(self):
         optimized_set = set()
         for fqdn, sld in self.fqdn_dict.items():
@@ -555,6 +571,7 @@ class OptimizeConfig(Config):
             self.ip_subnet_dict.update(i.ip_subnet_dict)
             self.fqdn_dict.update(i.fqdn_dict)
             self.sld_dict.update(i.sld_dict)
+        self._remove_subdomains()
         self._optimize_fqdn_dict()
         self._optimize_ip_dict()
         self._group_ip_ranges()
@@ -666,6 +683,16 @@ class ZiFQDN(BlackListParser):
         self.site_encoding = self.ZI_ENCODING
         self.fields_separator = ";"
         self.ips_separator = "|"
+        # self.decomp_obj = zlib.decompressobj(wbits=47)
+
+    # def prepare_data(self, url):
+    #     """
+    #     for https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
+    #     """
+    #     for chunk in self._download_data(url):
+    #         if chunk:
+    #             data = self.decomp_obj.decompress(chunk)
+    #             yield data
 
     def parser_func(self):
         for url in self.url:
@@ -693,7 +720,6 @@ class ZiIp(ZiFQDN):
                 for i in entry_list[0].split(self.ips_separator):
                     self.ip_value_processing(i)
 
-
 class AfFQDN(BlackListParser):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

ruantiblock/Makefile

@@ -5,7 +5,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock
-PKG_VERSION:=2.1.2
+PKG_VERSION:=2.1.3
 PKG_RELEASE:=1
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 

ruantiblock/files/usr/bin/ruantiblock

@@ -73,7 +73,7 @@ export ENABLE_LOGGING=1
 ### Вывод дополнительных сообщений в лог (0 - выкл, 1, 2)
 export DEBUG=0
 ### Файл для вывода некоторых отладочных сообщений
-export DEBUG_FILE="/tmp/ruanliblock.debug"
+export DEBUG_FILE="/tmp/${NAME}.debug"
 ### Html-страница с инфо о текущем статусе (0 - выкл, 1 - вкл) (не используется в OpenWrt)
 export ENABLE_HTML_INFO=0
 ### Максимальное кол-во элементов списка nftables
@@ -998,7 +998,7 @@ GetMainInstanceEntries() {
 }
 
 GetBlacklistFiles() {
-    local _return_code=1 _user_entries_ret_code=1
+    local _return_code=0 _user_entries_ret_code=1
     AddBypassEntries
     GetMainInstanceEntries
     case $? in