Refactoring: ruantiblock-mod-lua, ruantiblock-mod-py.

autoinstall/current/autoinstall.sh

@@ -9,8 +9,8 @@ LUA_MODULE=0
 LUCI_APP=1
 
 OWRT_VERSION="current"
-RUAB_VERSION="1.4-1"
+RUAB_VERSION="1.4-2"
-RUAB_MOD_LUA_VERSION="1.4-0"
+RUAB_MOD_LUA_VERSION="1.4-2"
 RUAB_LUCI_APP_VERSION="1.4-2"
 BASE_URL="https://raw.githubusercontent.com/gSpotx2f/packages-openwrt/master"
 PKG_DIR="/tmp"

ruantiblock-mod-lua/Makefile

@@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock-mod-lua
 PKG_VERSION:=1.4
-PKG_RELEASE:=0
+PKG_RELEASE:=2
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 
 include $(INCLUDE_DIR)/package.mk

ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua

@@ -1,7 +1,7 @@
 #!/usr/bin/env lua
 
 --[[
- (с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
+ (с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
 
  lua == 5.1
 --]]
@@ -66,6 +66,7 @@ local Config = Class(nil, {
         ["UPDATE_STATUS_FILE"] = true,
         ["RBL_ALL_URL"] = true,
         ["RBL_IP_URL"] = true,
+        ["RBL_DPI_URL"] = true,
         ["ZI_ALL_URL"] = true,
         ["AF_IP_URL"] = true,
         ["AF_FQDN_URL"] = true,
@@ -92,7 +93,7 @@ local Config = Class(nil, {
     encoding = "UTF-8",
     site_encoding = "",
     http_send_headers = {
-        --["User-Agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",
+        ["User-Agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0",
     },
 })
 Config.wget_user_agent = (Config.http_send_headers["User-Agent"]) and ' -U "' .. Config.http_send_headers["User-Agent"] .. '"' or ''
@@ -211,7 +212,180 @@ else
     error("Config.ICONV_TYPE should be either 'lua' or 'standalone'")
 end
 
------------------------------ Summarize ------------------------------
+------------------------------ Classes -------------------------------
+
+local BlackListParser = Class(Config, {
+    ip_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?",
+    cidr_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?/%d%d?",
+    fqdn_pattern = "[a-z0-9_%.%-]-[a-z0-9_%-]+%.[a-z0-9%.%-]+",
+    url = "http://127.0.0.1",
+    records_separator = "\n",
+})
+
+function BlackListParser:new(t)
+    -- extended instance constructor
+    local instance = self(t)
+    instance.url = instance["url"] or self.url
+    instance.records_separator = instance["records_separator"] or self.records_separator
+    instance.site_encoding = instance["site_encoding"] or self.site_encoding
+    instance.cidr_count = 0
+    instance.cidr_table = {}
+    instance.ip_subnet_table = {}
+    instance.ip_records_count = 0
+    instance.ip_count = 0
+    instance.ip_table = {}
+    instance.sld_table = {}
+    instance.fqdn_count = 0
+    instance.fqdn_records_count = 0
+    instance.fqdn_table = {}
+    instance.iconv_handler = iconv and iconv.open(instance.encoding, instance.site_encoding) or nil
+    return instance
+end
+
+function BlackListParser:convert_encoding(input)
+    local output
+    if self.ICONV_TYPE == "lua" and self.iconv_handler then
+        output = self.iconv_handler:iconv(input)
+    elseif self.ICONV_TYPE == "standalone" and self.ICONV_CMD then
+        local iconv_handler = assert(io.popen('printf \'' .. input .. '\' | ' .. self.ICONV_CMD .. ' -f "' .. self.site_encoding .. '" -t "' .. self.encoding .. '"', 'r'))
+        output = iconv_handler:read("*a")
+        iconv_handler:close()
+    end
+    return (output)
+end
+
+function BlackListParser:convert_to_punycode(input)
+    if self.site_encoding and self.site_encoding ~= "" then
+        input = self:convert_encoding(input)
+    end
+    return input and (idn.encode(input))
+end
+
+function BlackListParser:check_filter(str, filter_patterns, reverse)
+    if filter_patterns and str then
+        for pattern in pairs(filter_patterns) do
+            if str:match(pattern) then
+                return not reverse
+            end
+        end
+    end
+    return reverse
+end
+
+function BlackListParser:get_subnet(ip)
+    return ip:match("^(%d+%.%d+%.%d+%.)%d+$")
+end
+
+function BlackListParser:ip_value_processing(value)
+    if value and value ~= "" then
+        for ip_entry in value:gmatch(self.ip_pattern .. "/?%d?%d?") do
+            if not self.BLLIST_IP_FILTER or (self.BLLIST_IP_FILTER and not self:check_filter(ip_entry, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE)) then
+                if ip_entry:match("^" .. self.ip_pattern .. "$") and not self.ip_table[ip_entry] then
+                    local subnet = self:get_subnet(ip_entry)
+                    if subnet and (self.BLLIST_GR_EXCLUDED_NETS[subnet] or ((not self.BLLIST_IP_LIMIT or self.BLLIST_IP_LIMIT == 0) or (not self.ip_subnet_table[subnet] or self.ip_subnet_table[subnet] <= self.BLLIST_IP_LIMIT))) then
+                        self.ip_table[ip_entry] = subnet
+                        self.ip_subnet_table[subnet] = (self.ip_subnet_table[subnet] or 0) + 1
+                        self.ip_count = self.ip_count + 1
+                    end
+                elseif ip_entry:match("^" .. self.cidr_pattern .. "$") and not self.cidr_table[ip_entry] then
+                    self.cidr_table[ip_entry] = true
+                    self.cidr_count = self.cidr_count + 1
+                end
+            end
+        end
+    end
+end
+
+function BlackListParser:get_sld(fqdn)
+    return fqdn:match("^[a-z0-9_%.%-]-([a-z0-9_%-]+%.[a-z0-9%-]+)$")
+end
+
+function BlackListParser:fqdn_value_processing(value)
+    value = value:gsub("%*%.", ""):gsub("%.$", ""):lower()
+    if self.BLLIST_STRIP_WWW then
+        value = value:gsub("^www[0-9]?%.", "")
+    end
+    if not self.BLLIST_FQDN_FILTER or (self.BLLIST_FQDN_FILTER and not self:check_filter(value, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)) then
+        if value:match("^" .. self.fqdn_pattern .. "$") then
+        elseif self.BLLIST_ENABLE_IDN and value:match("^[^\\/&%?]-[^\\/&%?%.]+%.[^\\/&%?%.]+%.?$") then
+            value = self:convert_to_punycode(value)
+            if not value then
+                return false
+            end
+        else
+            return false
+        end
+        local sld = self:get_sld(value)
+        if sld and (self.BLLIST_GR_EXCLUDED_SLD[sld] or ((not self.BLLIST_SD_LIMIT or self.BLLIST_SD_LIMIT == 0) or (not self.sld_table[sld] or self.sld_table[sld] < self.BLLIST_SD_LIMIT))) then
+            self.fqdn_table[value] = sld
+            self.sld_table[sld] = (self.sld_table[sld] or 0) + 1
+            self.fqdn_count = self.fqdn_count + 1
+        end
+    end
+    return true
+end
+
+function BlackListParser:parser_func()
+    -- Must be overridden by a subclass
+    error("Method BlackListParser:parser_func() must be overridden by a subclass!")
+end
+
+function BlackListParser:chunk_buffer()
+    local buff = ""
+    local ret_value = ""
+    local last_chunk
+    return function(chunk)
+        if last_chunk then
+            return nil
+        end
+        if chunk then
+            buff = buff .. chunk
+            local last_rs_position = select(2, buff:find("^.*" .. self.records_separator))
+            if last_rs_position then
+                ret_value = buff:sub(1, last_rs_position)
+                buff = buff:sub((last_rs_position + 1), -1)
+            else
+                ret_value = ""
+            end
+        else
+            ret_value = buff
+            last_chunk = true
+        end
+        return (ret_value)
+    end
+end
+
+function BlackListParser:get_http_data(url)
+    local ret_val, ret_code, ret_headers
+    local http_module = url:match("^https") and https or http
+    if http_module then
+        local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
+        ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers}
+        if not ret_val or ret_code ~= 200 then
+            ret_val = nil
+            print(string.format("Connection error! (%s) URL: %s", ret_code, url))
+        end
+    else
+        local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
+        ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink)
+    end
+    return (ret_val == 1) and true or false
+end
+
+function BlackListParser:run()
+    local return_code = 0
+    if self:get_http_data(self.url) then
+        if (self.fqdn_count + self.ip_count + self.cidr_count) > self.BLLIST_MIN_ENTRIES then
+            return_code = 0
+        else
+            return_code = 2
+        end
+    else
+        return_code = 1
+    end
+    return return_code
+end
+
 
 local Summarize = {
     HOSTS_LIMIT = 0,
@@ -407,127 +581,27 @@ function Summarize:summarize_nets(cidr_list, modify_raw_list)
     )
 end
 
------------------------------- Classes -------------------------------
 
-local BlackListParser = Class(Config, {
+local OptimizeConfig = Class(Config, {
-    ip_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?",
+    parsers_list = {},
-    cidr_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?/%d%d?",
-    fqdn_pattern = "[a-z0-9_%.%-]-[a-z0-9_%-]+%.[a-z0-9%.%-]",
-    url = "http://127.0.0.1",
-    records_separator = "\n",
-    ips_separator = "|",
 })
 
-function BlackListParser:new(t)
+function OptimizeConfig:new(t)
     -- extended instance constructor
     local instance = self(t)
-    instance.url = instance["url"] or self.url
+    instance.parsers_list = instance.parsers_list or self.parsers_list
-    instance.records_separator = instance["records_separator"] or self.records_separator
-    instance.ips_separator = instance["ips_separator"] or self.ips_separator
-    instance.site_encoding = instance["site_encoding"] or self.site_encoding
-    instance.ip_subnet_table = {}
     instance.cidr_count = 0
+    instance.cidr_table = {}
+    instance.ip_subnet_table = {}
     instance.ip_records_count = 0
-    instance.ip_count = 0
+    instance.ip_table = {}
-    instance.fqdn_count = 0
     instance.sld_table = {}
     instance.fqdn_records_count = 0
-    instance.cidr_table = {}
-    instance.ip_table = {}
     instance.fqdn_table = {}
-    instance.iconv_handler = iconv and iconv.open(instance.encoding, instance.site_encoding) or nil
     return instance
 end
 
-function BlackListParser:convert_encoding(input)
+function OptimizeConfig:_optimize_ip_table()
-    local output
-    if self.ICONV_TYPE == "lua" and self.iconv_handler then
-        output = self.iconv_handler:iconv(input)
-    elseif self.ICONV_TYPE == "standalone" and self.ICONV_CMD then
-        local iconv_handler = assert(io.popen('printf \'' .. input .. '\' | ' .. self.ICONV_CMD .. ' -f "' .. self.site_encoding .. '" -t "' .. self.encoding .. '"', 'r'))
-        output = iconv_handler:read("*a")
-        iconv_handler:close()
-    end
-    return (output)
-end
-
-function BlackListParser:convert_to_punycode(input)
-    if self.site_encoding and self.site_encoding ~= "" then
-        input = self:convert_encoding(input)
-    end
-    return input and (idn.encode(input))
-end
-
-function BlackListParser:check_filter(str, filter_patterns, reverse)
-    if filter_patterns and str then
-        for pattern in pairs(filter_patterns) do
-            if str:match(pattern) then
-                return not reverse
-            end
-        end
-    end
-    return reverse
-end
-
-function BlackListParser:get_subnet(ip)
-    return ip:match("^(%d+%.%d+%.%d+%.)%d+$")
-end
-
-function BlackListParser:fill_ip_tables(val)
-    if val and val ~= "" then
-        for ip_entry in val:gmatch(self.ip_pattern .. "/?%d?%d?") do
-            if not self.BLLIST_IP_FILTER or (self.BLLIST_IP_FILTER and not self:check_filter(ip_entry, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE)) then
-                if ip_entry:match("^" .. self.ip_pattern .. "$") and not self.ip_table[ip_entry] then
-                    local subnet = self:get_subnet(ip_entry)
-                    if subnet and (self.BLLIST_GR_EXCLUDED_NETS[subnet] or ((not self.BLLIST_IP_LIMIT or self.BLLIST_IP_LIMIT == 0) or (not self.ip_subnet_table[subnet] or self.ip_subnet_table[subnet] <= self.BLLIST_IP_LIMIT))) then
-                        self.ip_table[ip_entry] = subnet
-                        self.ip_subnet_table[subnet] = (self.ip_subnet_table[subnet] or 0) + 1
-                        self.ip_count = self.ip_count + 1
-                    end
-                elseif ip_entry:match("^" .. self.cidr_pattern .. "$") and not self.cidr_table[ip_entry] then
-                    self.cidr_table[ip_entry] = true
-                    self.cidr_count = self.cidr_count + 1
-                end
-            end
-        end
-    end
-end
-
-function BlackListParser:get_sld(fqdn)
-    return fqdn:match("^[a-z0-9_%.%-]-([a-z0-9_%-]+%.[a-z0-9%-]+)$")
-end
-
-function BlackListParser:fill_domain_tables(val)
-    val = val:gsub("%*%.", ""):gsub("%.$", ""):lower()
-    if self.BLLIST_STRIP_WWW then
-        val = val:gsub("^www[0-9]?%.", "")
-    end
-    if not self.BLLIST_FQDN_FILTER or (self.BLLIST_FQDN_FILTER and not self:check_filter(val, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)) then
-        if val:match("^" .. self.fqdn_pattern .. "+$") then
-        elseif self.BLLIST_ENABLE_IDN and val:match("^[^\\/&%?]-[^\\/&%?%.]+%.[^\\/&%?%.]+%.?$") then
-            val = self:convert_to_punycode(val)
-            if not val then
-                return false
-            end
-        else
-            return false
-        end
-        local sld = self:get_sld(val)
-        if sld and (self.BLLIST_GR_EXCLUDED_SLD[sld] or ((not self.BLLIST_SD_LIMIT or self.BLLIST_SD_LIMIT == 0) or (not self.sld_table[sld] or self.sld_table[sld] < self.BLLIST_SD_LIMIT))) then
-            self.fqdn_table[val] = sld
-            self.sld_table[sld] = (self.sld_table[sld] or 0) + 1
-            self.fqdn_count = self.fqdn_count + 1
-        end
-    end
-    return true
-end
-
-function BlackListParser:sink()
-    -- Must be overridden by a subclass
-    error("Method BlackListParser:sink() must be overridden by a subclass!")
-end
-
-function BlackListParser:optimize_ip_table()
     local optimized_table = {}
     for ipaddr, subnet in pairs(self.ip_table) do
         if self.ip_subnet_table[subnet] then
@@ -544,8 +618,7 @@ function BlackListParser:optimize_ip_table()
     self.ip_table = optimized_table
 end
 
-
+function OptimizeConfig:_optimize_fqdn_table()
-function BlackListParser:optimize_fqdn_table()
     local optimized_table = {}
     if self.BLLIST_GR_EXCLUDED_MASKS and #self.BLLIST_GR_EXCLUDED_MASKS > 0 then
         for sld in pairs(self.sld_table) do
@@ -571,19 +644,60 @@ function BlackListParser:optimize_fqdn_table()
     self.fqdn_table = optimized_table
 end
 
-function BlackListParser:group_ip_ranges()
+function OptimizeConfig:_group_ip_ranges()
     for i in Summarize:summarize_ip_ranges(self.ip_table, true) do
         self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true
     end
 end
 
-function BlackListParser:group_cidr_ranges()
+function OptimizeConfig:_group_cidr_ranges()
     for i in Summarize:summarize_nets(self.cidr_table, true) do
         self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true
     end
 end
 
-function BlackListParser:write_ipset_config()
+function OptimizeConfig:_union(t1, t2)
+    local new_items = 0
+    for k, v in pairs(t2) do
+        if t1[k] == nil then
+            t1[k] = v
+            new_items = new_items + 1
+        end
+    end
+    return new_items
+end
+
+function OptimizeConfig:optimize()
+    for _, i in ipairs(self.parsers_list) do
+        self:_union(self.cidr_table, i.cidr_table)
+        self:_union(self.ip_table, i.ip_table)
+        self:_union(self.ip_subnet_table, i.ip_subnet_table)
+        self:_union(self.fqdn_table, i.fqdn_table)
+        self:_union(self.sld_table, i.sld_table)
+    end
+    self:_optimize_fqdn_table()
+    self:_optimize_ip_table()
+    if self.BLLIST_SUMMARIZE_IP then
+        self:_group_ip_ranges()
+    end
+    if self.BLLIST_SUMMARIZE_CIDR then
+        self:_group_cidr_ranges()
+    end
+end
+
+
+local WriteConfigFiles = Class(Config, {})
+
+function WriteConfigFiles:new(t)
+    -- extended instance constructor
+    local instance = self(t)
+    instance.cidr_count = 0
+    instance.ip_count = 0
+    instance.fqdn_count = 0
+    return instance
+end
+
+function WriteConfigFiles:write_ipset_config(ip_table, cidr_table)
     local file_handler = assert(io.open(self.IP_DATA_FILE, "w"), "Could not open nftset config")
     for _, v in ipairs({ self.NFTSET_CIDR, self.NFTSET_IP }) do
         file_handler:write(string.format("flush set %s %s\n", self.NFT_TABLE, v))
@@ -592,9 +706,9 @@ function BlackListParser:write_ipset_config()
         string.format("table %s {\n%s", self.NFT_TABLE, self.NFTSET_CIDR_CFG)
     )
     local c = 0
-    if next(self.cidr_table) then
+    if next(cidr_table) then
         file_handler:write("elements={")
-        for cidr in pairs(self.cidr_table) do
+        for cidr in pairs(cidr_table) do
             file_handler:write(string.format("%s,", cidr))
             c = c + 1
         end
@@ -605,123 +719,59 @@ function BlackListParser:write_ipset_config()
         string.format("}\n%s", self.NFTSET_IP_CFG)
     )
     local i = 0
-    if next(self.ip_table) then
+    if next(ip_table) then
         file_handler:write("elements={")
-        for ipaddr in pairs(self.ip_table) do
+        for ipaddr in pairs(ip_table) do
             file_handler:write(string.format("%s,", ipaddr))
             i = i + 1
         end
         file_handler:write("};")
     end
-    self.ip_records_count = i
+    self.ip_count = i
     file_handler:write("}\n}\n")
     file_handler:close()
 end
 
-function BlackListParser:write_dnsmasq_config()
+function WriteConfigFiles:write_dnsmasq_config(fqdn_table)
     local file_handler = assert(io.open(self.DNSMASQ_DATA_FILE, "w"), "Could not open dnsmasq config")
-    for fqdn in pairs(self.fqdn_table) do
+    local i = 0
+    for fqdn in pairs(fqdn_table) do
         if self.BLLIST_ALT_NSLOOKUP then
             file_handler:write(string.format("server=/%s/%s\n", fqdn, self.BLLIST_ALT_DNS_ADDR))
         end
         file_handler:write(string.format("nftset=/%s/%s#%s\n", fqdn, self.NFT_TABLE_DNSMASQ, self.NFTSET_DNSMASQ))
+        i = i + 1
     end
+    self.fqdn_count = i
     file_handler:close()
 end
 
-function BlackListParser:write_update_status()
+function WriteConfigFiles:write_update_status_file()
     local file_handler = assert(io.open(self.UPDATE_STATUS_FILE, "w"), "Could not open 'update_status' file")
-    file_handler:write(string.format("%d %d %d", self.cidr_count, self.ip_records_count, self.fqdn_records_count))
+    file_handler:write(string.format("%d %d %d", self.cidr_count, self.ip_count, self.fqdn_count))
     file_handler:close()
 end
 
-function BlackListParser:chunk_buffer()
+-- Parser subclasses
-    local buff = ""
-    local ret_value = ""
-    local last_chunk
-    return function(chunk)
-        if last_chunk then
-            return nil
-        end
-        if chunk then
-            buff = buff .. chunk
-            local last_rs_position = select(2, buff:find("^.*" .. self.records_separator))
-            if last_rs_position then
-                ret_value = buff:sub(1, last_rs_position)
-                buff = buff:sub((last_rs_position + 1), -1)
-            else
-                ret_value = ""
-            end
-        else
-            ret_value = buff
-            last_chunk = true
-        end
-        return (ret_value)
-    end
-end
 
-function BlackListParser:get_http_data(url)
+local function ip_parser_func(self)
-    local ret_val, ret_code, ret_headers
-    local http_module = url:match("^https") and https or http
-    if http_module then
-        local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:sink())
-        ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers}
-        if not ret_val or ret_code ~= 200 then
-            ret_val = nil
-            print(string.format("Connection error! (%s) URL: %s", ret_code, url))
-        end
-    else
-        local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:sink())
-        ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink)
-    end
-    return (ret_val == 1) and true or false
-end
-
-function BlackListParser:run()
-    local return_code = 0
-    if self:get_http_data(self.url) then
-        if (self.fqdn_count + self.ip_count + self.cidr_count) > self.BLLIST_MIN_ENTRIES then
-            self:optimize_fqdn_table()
-            self:optimize_ip_table()
-            if self.BLLIST_SUMMARIZE_IP then
-                self:group_ip_ranges()
-            end
-            if self.BLLIST_SUMMARIZE_CIDR then
-                self:group_cidr_ranges()
-            end
-            self:write_ipset_config()
-            self:write_dnsmasq_config()
-            return_code = 0
-        else
-            return_code = 2
-        end
-    else
-        return_code = 1
-    end
-    self:write_update_status()
-    return return_code
-end
-
--- Subclasses
-
-local function ip_sink(self)
     return function(chunk)
         if chunk and chunk ~= "" then
             for ip_string in chunk:gmatch(self.ip_string_pattern) do
-                self:fill_ip_tables(ip_string)
+                self:ip_value_processing(ip_string)
             end
         end
         return true
     end
 end
 
-local function fqdn_sink_func(self, ip_str, fqdn_str)
+local function fqdn_parser_func(self, ip_str, fqdn_str)
     if #fqdn_str > 0 and not fqdn_str:match("^" .. self.ip_pattern .. "$") then
-        if self:fill_domain_tables(fqdn_str) then
+        if self:fqdn_value_processing(fqdn_str) then
             return true
         end
     end
-    self:fill_ip_tables(ip_str)
+    self:ip_value_processing(ip_str)
 end
 
     -- rublacklist.net
@@ -729,14 +779,13 @@ end
 local Rbl = Class(BlackListParser, {
     url = Config.RBL_ALL_URL,
     records_separator = '%{"appearDate": ',
-    ips_separator = ", ",
 })
 
-function Rbl:sink()
+function Rbl:parser_func()
     return function(chunk)
         if chunk and chunk ~= "" then
             for fqdn_str, ip_str in chunk:gmatch('"domains": %["?(.-)"?%].-"ips": %[([a-f0-9/.:", ]*)%].-') do
-                fqdn_sink_func(self, ip_str, fqdn_str)
+                fqdn_parser_func(self, ip_str, fqdn_str)
             end
         end
         return true
@@ -747,9 +796,28 @@ local RblIp = Class(Rbl, {
     url = Config.RBL_IP_URL,
     records_separator = ",",
     ip_string_pattern = "([a-f0-9/.:]+)",
-    sink = ip_sink,
+    parser_func = ip_parser_func,
 })
 
+local RblDPI = Class(BlackListParser, {
+    url = Config.RBL_DPI_URL,
+    BLLIST_MIN_ENTRIES = 0,
+    records_separator = '%{"domains"',
+})
+
+function RblDPI:parser_func()
+    return function(chunk)
+        if chunk and chunk ~= "" then
+            for fqdn_list in chunk:gmatch(': %[(.-)%]') do
+                for fqdn_str in fqdn_list:gmatch(self.fqdn_pattern) do
+                    self:fqdn_value_processing(fqdn_str)
+                end
+            end
+        end
+        return true
+    end
+end
+
     -- zapret-info
 
 local Zi = Class(BlackListParser, {
@@ -757,11 +825,11 @@ local Zi = Class(BlackListParser, {
     site_encoding = Config.ZI_ENCODING,
 })
 
-function Zi:sink()
+function Zi:parser_func()
     return function(chunk)
         if chunk and chunk ~= "" then
             for ip_str, fqdn_str in chunk:gmatch("([^;]-);([^;]-);.-" .. self.records_separator) do
-                fqdn_sink_func(self, ip_str, fqdn_str)
+                fqdn_parser_func(self, ip_str, fqdn_str)
             end
         end
         return true
@@ -770,7 +838,7 @@ end
 
 local ZiIp = Class(Zi, {
     ip_string_pattern = "([a-f0-9%.:/ |]+);.-\n",
-    sink = ip_sink,
+    parser_func = ip_parser_func,
 })
 
     -- antifilter
@@ -779,12 +847,12 @@ local Af = Class(BlackListParser, {
     url = Config.AF_FQDN_URL,
 })
 
-function Af:sink()
+function Af:parser_func()
     local entry_pattern = "((.-))" .. self.records_separator
     return function(chunk)
         if chunk and chunk ~= "" then
             for fqdn_str, ip_str in chunk:gmatch(entry_pattern) do
-                fqdn_sink_func(self, ip_str, fqdn_str)
+                fqdn_parser_func(self, ip_str, fqdn_str)
             end
         end
         return true
@@ -794,7 +862,7 @@ end
 local AfIp = Class(Af, {
     url = Config.AF_IP_URL,
     ip_string_pattern = "(.-)\n",
-    sink = ip_sink,
+    parser_func = ip_parser_func,
 })
 
     -- ruantiblock
@@ -821,7 +889,7 @@ function Ra:chunk_buffer()
     end
 end
 
-function Ra:sink()
+function Ra:parser_func()
     return function(chunk)
         if chunk and chunk ~= "" then
             self.current_file_handler:write(chunk)
@@ -850,17 +918,34 @@ local RaIp = Class(Ra, {
 
 ----------------------------- Main section ------------------------------
 
-local ctx_table = {
+local parsers_table = {
-    ["ip"] = {["rublacklist"] = RblIp, ["zapret-info"] = ZiIp, ["antifilter"] = AfIp, ["ruantiblock"] = RaIp},
+    ["ip"] = {["rublacklist"] = {RblIp}, ["zapret-info"] = {ZiIp}, ["antifilter"] = {AfIp}, ["ruantiblock"] = {RaIp}},
-    ["fqdn"] = {["rublacklist"] = Rbl, ["zapret-info"] = Zi, ["antifilter"] = Af, ["ruantiblock"] = Ra},
+    ["fqdn"] = {["rublacklist"] = {Rbl, RblDPI}, ["zapret-info"] = {Zi}, ["antifilter"] = {Af}, ["ruantiblock"] = {Ra}},
 }
 
-local return_code = 1
+local ret_list = {}
-local ctx = ctx_table[Config.BLLIST_MODE] and ctx_table[Config.BLLIST_MODE][Config.BLLIST_SOURCE]
+local parser_classes = parsers_table[Config.BLLIST_MODE] and parsers_table[Config.BLLIST_MODE][Config.BLLIST_SOURCE]
-if ctx then
+if parser_classes then
-    return_code = ctx:new():run()
+    local parser_instances = {}
+    for _, i in ipairs(parser_classes) do
+        parser_instances[#parser_instances + 1] = i:new()
+    end
+    for _, i in ipairs(parser_instances) do
+        ret_list[i:run()] = true
+    end
+    return_sum = 0
+    for i, _ in pairs(ret_list) do
+        return_sum = return_sum + i
+    end
+    if return_sum == 0 and Config.BLLIST_SOURCE ~= "ruantiblock" then
+        local oc_obj = OptimizeConfig:new({parsers_list = parser_instances})
+        oc_obj:optimize()
+        local write_cfg_obj = WriteConfigFiles:new()
+        write_cfg_obj:write_dnsmasq_config(oc_obj.fqdn_table)
+        write_cfg_obj:write_ipset_config(oc_obj.ip_table, oc_obj.cidr_table)
+        write_cfg_obj:write_update_status_file()
+    end
 else
-    error("Wrong configuration! (Config.BLLIST_MODE or Config.BLLIST_SOURCE)")
+    error("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)")
 end
-
+os.exit(ret_list[1] and 1 or (ret_list[2] and 2 or 0))
-os.exit(return_code)

ruantiblock-mod-py/Makefile

@@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock-mod-py
 PKG_VERSION:=1.4
-PKG_RELEASE:=0
+PKG_RELEASE:=2
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 
 include $(INCLUDE_DIR)/package.mk

ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 """
-(с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
+(с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
 
  Python >= 3.6
 """
@@ -50,6 +50,7 @@ class Config:
         "UPDATE_STATUS_FILE",
         "RBL_ALL_URL",
         "RBL_IP_URL",
+        "RBL_DPI_URL",
         "ZI_ALL_URL",
         "AF_IP_URL",
         "AF_FQDN_URL",
@@ -120,98 +121,6 @@ class Config:
             cls._load_filter(file_path or cls.BLLIST_IP_FILTER_FILE, cls.BLLIST_IP_FILTER_PATTERNS)
 
 
-class Summarize:
-    HOSTS_LIMIT = 0
-    NETS_LIMIT = 0
-
-    @staticmethod
-    def _sort_ip_func(e):
-        return IPv4Address(e)
-
-    @classmethod
-    def _group_ip_ranges(cls, ip_list, raw_list=None):
-
-        def remove_items(start, end):
-            for ip in range(int(start), int(end) + 1):
-                raw_list.remove(str(IPv4Address(ip)))
-
-        start = end = None
-        hosts = 1
-        for ip in ip_list:
-            ip_obj = IPv4Address(ip)
-            if end and (end + 1) == ip_obj:
-                hosts += 1
-            else:
-                if hosts > 1 and hosts >= cls.HOSTS_LIMIT:
-                    if raw_list:
-                        remove_items(start, end)
-                    yield start, end
-                start = ip_obj
-                hosts = 1
-            end = ip_obj
-        else:
-            if hosts > 1 and hosts >= HOSTS_LIMIT:
-                if raw_list:
-                    remove_items(start, end)
-                yield start, end
-
-    @classmethod
-    def summarize_ip_ranges(cls, ip_list, modify_raw_list=False):
-        for s, e in cls._group_ip_ranges(sorted(ip_list, key=cls._sort_ip_func),
-                                    modify_raw_list and ip_list):
-            for i in summarize_address_range(s, e):
-                if i.prefixlen == 32:
-                    if modify_raw_list:
-                        if type(ip_list) == set:
-                            ip_list.add(i.network_address)
-                        else:
-                            ip_list.append(i.network_address)
-                else:
-                    yield i
-
-    @staticmethod
-    def _sort_net_func(e):
-        return IPv4Network(e)
-
-    @classmethod
-    def _group_nets(cls, cidr_list, raw_list=None):
-
-        def remove_items(start, end):
-            for ip in range(int(start), int(end) + 1, 256):
-                raw_list.remove(str(IPv4Address(ip)) + "/24")
-
-        start = end = curr_super_net = None
-        nets = 1
-        for net in cidr_list:
-            net_obj = IPv4Network(net)
-            prefix_len = net_obj.prefixlen
-            if prefix_len == 24:
-                address = net_obj.network_address
-                super_net = net_obj.supernet(new_prefix=16)
-                if end and super_net == curr_super_net and (end + 256) == address:
-                    nets += 1
-                else:
-                    if nets > 1 and nets >= cls.NETS_LIMIT:
-                        if raw_list:
-                            remove_items(start, end)
-                        yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255))
-                    start = address
-                    curr_super_net = super_net
-                    nets = 1
-                end = address
-        else:
-            if nets > 1 and nets >= cls.NETS_LIMIT:
-                if raw_list:
-                    remove_items(start, end)
-                yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255))
-
-    @classmethod
-    def summarize_nets(cls, cidr_list):
-        for i in cls._group_nets(sorted(cidr_list, key=cls._sort_net_func), cidr_list):
-            for j in i:
-                yield j
-
-
 class ParserError(Exception):
     def __init__(self, reason=None):
         super().__init__(reason)
@@ -235,16 +144,16 @@ class BlackListParser(Config):
         self.www_pattern = re.compile(r"^www[0-9]?[.]")
         self.cyr_pattern = re.compile(r"[а-яё]", re.U)
         self.cidr_set = set()
-        self.ip_set = {}
+        self.ip_dict = {}
         self.ip_subnet_dict = {}
-        self.fqdn_set = {}
+        self.fqdn_dict = {}
         self.sld_dict = {}
         self.cidr_count = 0
         self.ip_count = 0
         self.output_fqdn_count = 0
         self.ssl_unverified = False
         self.send_headers_dict = {
-            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",
+            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0",
         }
         ### Proxies (ex.: self.proxies = {"http": "http://192.168.0.1:8080", "https": "http://192.168.0.1:8080"})
         self.proxies = None
@@ -252,8 +161,6 @@ class BlackListParser(Config):
         self.data_chunk = 2048
         self.url = "http://127.0.0.1"
         self.records_separator = "\n"
-        self.fields_separator = ";"
-        self.ips_separator = "|"
         self.default_site_encoding = "utf-8"
         self.site_encoding = self.default_site_encoding
 
@@ -343,22 +250,21 @@ class BlackListParser(Config):
         regexp_obj = self.ip_pattern.fullmatch(ip_addr)
         return regexp_obj.group(1) if regexp_obj else None
 
-    def ip_field_processing(self, string):
+    def ip_value_processing(self, value):
-        for i in string.split(self.ips_separator):
         if self.BLLIST_IP_FILTER and self._check_filter(
-                i, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE):
+            value, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE):
-                continue
+            return
-            if self.ip_pattern.fullmatch(i) and i not in self.ip_set:
+        if self.ip_pattern.fullmatch(value) and value not in self.ip_dict:
-                subnet = self._get_subnet(i)
+            subnet = self._get_subnet(value)
             if subnet in self.BLLIST_GR_EXCLUDED_NETS or (
                 not self.BLLIST_IP_LIMIT or (
                     subnet not in self.ip_subnet_dict or self.ip_subnet_dict[subnet] < self.BLLIST_IP_LIMIT
                 )
             ):
-                    self.ip_set[i] = subnet
+                self.ip_dict[value] = subnet
                 self.ip_subnet_dict[subnet] = (self.ip_subnet_dict.get(subnet) or 0) + 1
-            elif self.cidr_pattern.fullmatch(i) and i not in self.cidr_set:
+        elif self.cidr_pattern.fullmatch(value) and value not in self.cidr_set:
-                self.cidr_set.add(i)
+                self.cidr_set.add(value)
 
     def _convert_to_punycode(self, string):
         if self.cyr_pattern.search(string):
@@ -376,26 +282,26 @@ class BlackListParser(Config):
         regexp_obj = self.fqdn_pattern.fullmatch(fqdn)
         return regexp_obj.group(2) if regexp_obj else None
 
-    def fqdn_field_processing(self, string):
+    def fqdn_value_processing(self, value):
-        if self.ip_pattern.fullmatch(string):
+        if self.ip_pattern.fullmatch(value):
             raise FieldValueError()
-        string = string.strip("*.").lower()
+        value = value.strip("*.").lower()
         if self.BLLIST_STRIP_WWW:
-            string = self.www_pattern.sub("", string)
+            value = self.www_pattern.sub("", value)
         if not self.BLLIST_FQDN_FILTER or (
             self.BLLIST_FQDN_FILTER and not self._check_filter(
-                string, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)
+                value, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)
         ):
-            if self.fqdn_pattern.fullmatch(string):
+            if self.fqdn_pattern.fullmatch(value):
-                string = self._convert_to_punycode(string)
+                value = self._convert_to_punycode(value)
-                sld = self._get_sld(string)
+                sld = self._get_sld(value)
                 if sld in self.BLLIST_GR_EXCLUDED_SLD or (
                     not self.BLLIST_SD_LIMIT or (
                         sld not in self.sld_dict or self.sld_dict[sld] < self.BLLIST_SD_LIMIT
                     )
                 ):
                     self.sld_dict[sld] = (self.sld_dict.get(sld) or 0) + 1
-                    self.fqdn_set[string] = sld
+                    self.fqdn_dict[value] = sld
             else:
                 raise FieldValueError()
 
@@ -403,47 +309,11 @@ class BlackListParser(Config):
         """Must be overridden by a subclass"""
         raise NotImplementedError()
 
-    def _check_sld_masks(self, sld):
-        if self.BLLIST_GR_EXCLUDED_MASKS:
-            for pattern in self.BLLIST_GR_EXCLUDED_MASKS:
-                if re.fullmatch(pattern, sld):
-                    return True
-        return False
-
-    def _optimize_fqdn_set(self):
-        optimized_set = set()
-        for fqdn, sld in self.fqdn_set.items():
-            if sld and (fqdn == sld or sld not in self.fqdn_set) and self.sld_dict.get(sld):
-                if (not self._check_sld_masks(sld) and (
-                        self.BLLIST_SD_LIMIT and sld not in self.BLLIST_GR_EXCLUDED_SLD
-                )) and (self.sld_dict[sld] >= self.BLLIST_SD_LIMIT):
-                    record_value = sld
-                    del(self.sld_dict[sld])
-                else:
-                    record_value = fqdn
-                optimized_set.add(record_value)
-                self.output_fqdn_count += 1
-        self.fqdn_set = optimized_set
-
-    def _optimize_ip_set(self):
-        optimized_set = set()
-        for ip_addr, subnet in self.ip_set.items():
-            if subnet in self.ip_subnet_dict:
-                if subnet not in self.BLLIST_GR_EXCLUDED_NETS and (
-                    self.BLLIST_IP_LIMIT and self.ip_subnet_dict[subnet] >= self.BLLIST_IP_LIMIT
-                ):
-                    self.cidr_set.add(f"{subnet}0/24")
-                    del(self.ip_subnet_dict[subnet])
-                else:
-                    optimized_set.add(ip_addr)
-                    self.ip_count += 1
-        self.ip_set = optimized_set
-
     def _group_ip_ranges(self):
         if self.BLLIST_SUMMARIZE_IP:
-            for i in Summarize.summarize_ip_ranges(self.ip_set, True):
+            for i in Summarize.summarize_ip_ranges(self.ip_dict, True):
                 self.cidr_set.add(i.with_prefixlen)
-            self.ip_count = len(self.ip_set)
+            self.ip_count = len(self.ip_dict)
 
     def _group_cidr_ranges(self):
         if self.BLLIST_SUMMARIZE_CIDR:
@@ -457,17 +327,216 @@ class BlackListParser(Config):
         self.BLLIST_IP_FILTER_PATTERNS = self._compile_filter_patterns(self.BLLIST_IP_FILTER_PATTERNS)
         self.records_separator = bytes(self.records_separator, "utf-8")
         self.parser_func()
-        if (len(self.ip_set) + len(self.cidr_set) + len(self.fqdn_set)) >= self.BLLIST_MIN_ENTRIES:
+        if (len(self.ip_dict) + len(self.cidr_set) + len(self.fqdn_dict)) >= self.BLLIST_MIN_ENTRIES:
-            self._optimize_fqdn_set()
-            self._optimize_ip_set()
-            self._group_ip_ranges()
-            self._group_cidr_ranges()
             ret_value = 0
         else:
             ret_value = 2
         return ret_value
 
 
+class Summarize:
+    HOSTS_LIMIT = 0
+    NETS_LIMIT = 0
+
+    @staticmethod
+    def _sort_ip_func(e):
+        return IPv4Address(e)
+
+    @classmethod
+    def _group_ip_ranges(cls, ip_list, raw_list=None):
+        def remove_items(start, end):
+            for ip in range(int(start), int(end) + 1):
+                raw_list.remove(str(IPv4Address(ip)))
+
+        start = end = None
+        hosts = 1
+        for ip in ip_list:
+            ip_obj = IPv4Address(ip)
+            if end and (end + 1) == ip_obj:
+                hosts += 1
+            else:
+                if hosts > 1 and hosts >= cls.HOSTS_LIMIT:
+                    if raw_list:
+                        remove_items(start, end)
+                    yield start, end
+                start = ip_obj
+                hosts = 1
+            end = ip_obj
+        else:
+            if hosts > 1 and hosts >= HOSTS_LIMIT:
+                if raw_list:
+                    remove_items(start, end)
+                yield start, end
+
+    @classmethod
+    def summarize_ip_ranges(cls, ip_list, modify_raw_list=False):
+        for s, e in cls._group_ip_ranges(sorted(ip_list, key=cls._sort_ip_func),
+                                    modify_raw_list and ip_list):
+            for i in summarize_address_range(s, e):
+                if i.prefixlen == 32:
+                    if modify_raw_list:
+                        if type(ip_list) == set:
+                            ip_list.add(i.network_address)
+                        else:
+                            ip_list.append(i.network_address)
+                else:
+                    yield i
+
+    @staticmethod
+    def _sort_net_func(e):
+        return IPv4Network(e)
+
+    @classmethod
+    def _group_nets(cls, cidr_list, raw_list=None):
+        def remove_items(start, end):
+            for ip in range(int(start), int(end) + 1, 256):
+                raw_list.remove(str(IPv4Address(ip)) + "/24")
+
+        start = end = curr_super_net = None
+        nets = 1
+        for net in cidr_list:
+            net_obj = IPv4Network(net)
+            prefix_len = net_obj.prefixlen
+            if prefix_len == 24:
+                address = net_obj.network_address
+                super_net = net_obj.supernet(new_prefix=16)
+                if end and super_net == curr_super_net and (end + 256) == address:
+                    nets += 1
+                else:
+                    if nets > 1 and nets >= cls.NETS_LIMIT:
+                        if raw_list:
+                            remove_items(start, end)
+                        yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255))
+                    start = address
+                    curr_super_net = super_net
+                    nets = 1
+                end = address
+        else:
+            if nets > 1 and nets >= cls.NETS_LIMIT:
+                if raw_list:
+                    remove_items(start, end)
+                yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255))
+
+    @classmethod
+    def summarize_nets(cls, cidr_list):
+        for i in cls._group_nets(sorted(cidr_list, key=cls._sort_net_func), cidr_list):
+            for j in i:
+                yield j
+
+
+class OptimizeConfig(Config):
+    def __init__(self, parsers_list):
+        self.parsers_list = parsers_list
+        self.cidr_set = set()
+        self.ip_dict = {}
+        self.ip_subnet_dict = {}
+        self.fqdn_dict = {}
+        self.sld_dict = {}
+        self.cidr_count = 0
+        self.ip_count = 0
+        self.output_fqdn_count = 0
+
+    def _check_sld_masks(self, sld):
+        if self.BLLIST_GR_EXCLUDED_MASKS:
+            for pattern in self.BLLIST_GR_EXCLUDED_MASKS:
+                if re.fullmatch(pattern, sld):
+                    return True
+        return False
+
+    def _optimize_fqdn_dict(self):
+        optimized_set = set()
+        for fqdn, sld in self.fqdn_dict.items():
+            if sld and (fqdn == sld or sld not in self.fqdn_dict) and self.sld_dict.get(sld):
+                if (not self._check_sld_masks(sld) and (
+                        self.BLLIST_SD_LIMIT and sld not in self.BLLIST_GR_EXCLUDED_SLD
+                )) and (self.sld_dict[sld] >= self.BLLIST_SD_LIMIT):
+                    record_value = sld
+                    del(self.sld_dict[sld])
+                else:
+                    record_value = fqdn
+                optimized_set.add(record_value)
+                self.output_fqdn_count += 1
+        self.fqdn_dict = optimized_set
+
+    def _optimize_ip_dict(self):
+        optimized_set = set()
+        for ip_addr, subnet in self.ip_dict.items():
+            if subnet in self.ip_subnet_dict:
+                if subnet not in self.BLLIST_GR_EXCLUDED_NETS and (
+                    self.BLLIST_IP_LIMIT and self.ip_subnet_dict[subnet] >= self.BLLIST_IP_LIMIT
+                ):
+                    self.cidr_set.add(f"{subnet}0/24")
+                    del(self.ip_subnet_dict[subnet])
+                else:
+                    optimized_set.add(ip_addr)
+                    self.ip_count += 1
+        self.ip_dict = optimized_set
+
+    def _group_ip_ranges(self):
+        if self.BLLIST_SUMMARIZE_IP:
+            for i in Summarize.summarize_ip_ranges(self.ip_dict, True):
+                self.cidr_set.add(i.with_prefixlen)
+            self.ip_count = len(self.ip_dict)
+
+    def _group_cidr_ranges(self):
+        if self.BLLIST_SUMMARIZE_CIDR:
+            for i in Summarize.summarize_nets(self.cidr_set):
+                self.cidr_set.add(i.with_prefixlen)
+        self.cidr_count = len(self.cidr_set)
+
+    def optimize(self):
+        for i in self.parsers_list:
+            self.cidr_set |= i.cidr_set
+            self.ip_dict.update(i.ip_dict)
+            self.ip_subnet_dict.update(i.ip_subnet_dict)
+            self.fqdn_dict.update(i.fqdn_dict)
+            self.sld_dict.update(i.sld_dict)
+        self._optimize_fqdn_dict()
+        self._optimize_ip_dict()
+        self._group_ip_ranges()
+        self._group_cidr_ranges()
+
+
+class WriteConfigFiles(Config):
+    def __init__(self):
+        self.write_buffer = -1
+
+    def write_ipset_config(self, ip_dict, cidr_set):
+        with open(self.IP_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler:
+            for i in (self.NFTSET_CIDR, self.NFTSET_IP):
+                file_handler.write("flush set {} {}\n".format(self.NFT_TABLE, i))
+            file_handler.write(
+                "table {} {{\n{}".format(self.NFT_TABLE, self.NFTSET_CIDR_CFG)
+            )
+            if len(cidr_set) > 0:
+                file_handler.write("elements={")
+                for i in cidr_set:
+                    file_handler.write(f"{i},")
+                file_handler.write("};")
+            file_handler.write(
+                "}}\n{}".format(self.NFTSET_IP_CFG)
+            )
+            if len(ip_dict) > 0:
+                file_handler.write("elements={")
+                for i in ip_dict:
+                    file_handler.write(f"{i},")
+                file_handler.write("};")
+            file_handler.write("}\n}\n")
+
+    def write_dnsmasq_config(self, fqdn_dict):
+        with open(self.DNSMASQ_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler:
+            for fqdn in fqdn_dict:
+                file_handler.write(
+                    f"server=/{fqdn}/{self.BLLIST_ALT_DNS_ADDR}\nnftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n"
+                    if self.BLLIST_ALT_NSLOOKUP else
+                    f"nftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n")
+
+    def write_update_status_file(self, ip_count, cidr_count, fqdn_count):
+        with open(self.UPDATE_STATUS_FILE, "wt") as file_handler:
+            file_handler.write(
+                f"{cidr_count} {ip_count} {fqdn_count}")
+
+
 class RblFQDN(BlackListParser):
     def __init__(self):
         super().__init__()
@@ -484,12 +553,33 @@ class RblFQDN(BlackListParser):
             fqdn_string = res.group(1)
             if fqdn_string:
                 try:
-                    self.fqdn_field_processing(fqdn_string)
+                    self.fqdn_value_processing(fqdn_string)
                 except FieldValueError:
-                    self.ip_field_processing(ip_string)
+                    for i in ip_string.split(self.ips_separator):
+                        self.ip_value_processing(i)
             else:
-                self.ip_field_processing(ip_string)
+                for i in ip_string.split(self.ips_separator):
+                    self.ip_value_processing(i)
 
+class RblDPI(BlackListParser):
+    def __init__(self):
+        super().__init__()
+        self.url = self.RBL_DPI_URL
+        self.BLLIST_MIN_ENTRIES = 0
+        self.records_separator = '{"domains"'
+
+    def parser_func(self):
+        for entry in self._split_entries():
+            res = re.search(r': \[(.*?)\]', entry)
+            if not res:
+                continue
+            fqdn_string = res.group(1)
+            if fqdn_string:
+                for i in fqdn_string.split(', "'):
+                    try:
+                        self.fqdn_value_processing(i.strip('"'))
+                    except FieldValueError:
+                        pass
 
 class RblIp(BlackListParser):
     def __init__(self):
@@ -499,7 +589,7 @@ class RblIp(BlackListParser):
 
     def parser_func(self):
         for entry in self._split_entries():
-            self.ip_field_processing(re.sub(r'[\[\]" ]', "", entry))
+            self.ip_value_processing(re.sub(r'[\[\]" ]', "", entry))
 
 
 class ZiFQDN(BlackListParser):
@@ -507,6 +597,8 @@ class ZiFQDN(BlackListParser):
         super().__init__()
         self.url = self.ZI_ALL_URL
         self.site_encoding = self.ZI_ENCODING
+        self.fields_separator = ";"
+        self.ips_separator = "|"
 
     def parser_func(self):
         for entry in self._split_entries():
@@ -514,11 +606,13 @@ class ZiFQDN(BlackListParser):
             try:
                 if entry_list[1]:
                     try:
-                        self.fqdn_field_processing(entry_list[1])
+                        self.fqdn_value_processing(entry_list[1])
                     except FieldValueError:
-                        self.ip_field_processing(entry_list[0])
+                        for i in entry_list[0].split(self.ips_separator):
+                            self.ip_value_processing(i)
                 else:
-                    self.ip_field_processing(entry_list[0])
+                    for i in entry_list[0].split(self.ips_separator):
+                            self.ip_value_processing(i)
             except IndexError:
                 pass
 
@@ -527,7 +621,8 @@ class ZiIp(ZiFQDN):
     def parser_func(self):
         for entry in self._split_entries():
             entry_list = entry.split(self.fields_separator)
-            self.ip_field_processing(entry_list[0])
+            for i in entry_list[0].split(self.ips_separator):
+                self.ip_value_processing(i)
 
 
 class AfFQDN(BlackListParser):
@@ -538,9 +633,9 @@ class AfFQDN(BlackListParser):
     def parser_func(self):
         for entry in self._split_entries():
             try:
-                self.fqdn_field_processing(entry)
+                self.fqdn_value_processing(entry)
             except FieldValueError:
-                self.ip_field_processing(entry)
+                self.ip_value_processing(entry)
 
 
 class AfIp(BlackListParser):
@@ -550,7 +645,7 @@ class AfIp(BlackListParser):
 
     def parser_func(self):
         for entry in self._split_entries():
-            self.ip_field_processing(entry)
+            self.ip_value_processing(entry)
 
 
 class RaFQDN(BlackListParser):
@@ -586,64 +681,27 @@ class RaIp(RaFQDN):
         self.url_stat = self.RA_IP_STAT_URL
 
 
-class WriteConfigFiles(Config):
-    def __init__(self):
-        self.write_buffer = -1
-
-    def write_ipset_config(self, ip_set, cidr_set):
-        with open(self.IP_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler:
-            for i in (self.NFTSET_CIDR, self.NFTSET_IP):
-                file_handler.write("flush set {} {}\n".format(self.NFT_TABLE, i))
-            file_handler.write(
-                "table {} {{\n{}".format(self.NFT_TABLE, self.NFTSET_CIDR_CFG)
-            )
-            if len(cidr_set) > 0:
-                file_handler.write("elements={")
-                for i in cidr_set:
-                    file_handler.write(f"{i},")
-                file_handler.write("};")
-            file_handler.write(
-                "}}\n{}".format(self.NFTSET_IP_CFG)
-            )
-            if len(ip_set) > 0:
-                file_handler.write("elements={")
-                for i in ip_set:
-                    file_handler.write(f"{i},")
-                file_handler.write("};")
-            file_handler.write("}\n}\n")
-
-    def write_dnsmasq_config(self, fqdn_set):
-        with open(self.DNSMASQ_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler:
-            for fqdn in fqdn_set:
-                file_handler.write(
-                    f"server=/{fqdn}/{self.BLLIST_ALT_DNS_ADDR}\nnftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n"
-                    if self.BLLIST_ALT_NSLOOKUP else
-                    f"nftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n")
-
-    def write_update_status_file(self, ip_count, cidr_count, output_fqdn_count):
-        with open(self.UPDATE_STATUS_FILE, "wt") as file_handler:
-            file_handler.write(
-                f"{cidr_count} {ip_count} {output_fqdn_count}")
-
-
 if __name__ == "__main__":
     Config.load_environ_config()
     Config.load_fqdn_filter()
     Config.load_ip_filter()
-    ctx_dict = {
+    parsers_dict = {
-        "ip": {"rublacklist": RblIp, "zapret-info": ZiIp, "antifilter": AfIp, "ruantiblock": RaIp},
+        "ip": {"rublacklist": [RblIp], "zapret-info": [ZiIp], "antifilter": [AfIp], "ruantiblock": [RaIp]},
-        "fqdn": {"rublacklist": RblFQDN, "zapret-info": ZiFQDN, "antifilter": AfFQDN, "ruantiblock": RaFQDN},
+        "fqdn": {"rublacklist": [RblFQDN, RblDPI], "zapret-info": [ZiFQDN], "antifilter": [AfFQDN], "ruantiblock": [RaFQDN]},
     }
-    write_cfg_obj = WriteConfigFiles()
     try:
-        ctx = ctx_dict[Config.BLLIST_MODE][Config.BLLIST_SOURCE]()
+        parser_classes = parsers_dict[Config.BLLIST_MODE][Config.BLLIST_SOURCE]
     except KeyError:
-        print("Wrong configuration! (Config.BLLIST_MODE or Config.BLLIST_SOURCE)",
+        print("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)",
               file=sys.stderr)
         sys.exit(1)
-    ret_code = ctx.run()
+    parser_instances = [i() for i in parser_classes]
-    if ret_code == 0 and Config.BLLIST_SOURCE != "ruantiblock":
+    ret_list = [i.run() for i in parser_instances]
-        write_cfg_obj.write_dnsmasq_config(ctx.fqdn_set)
+    if sum(ret_list) == 0 and Config.BLLIST_SOURCE != "ruantiblock":
-        write_cfg_obj.write_ipset_config(ctx.ip_set, ctx.cidr_set)
+        oc_obj = OptimizeConfig(parser_instances)
-        write_cfg_obj.write_update_status_file(ctx.ip_count, ctx.cidr_count, ctx.output_fqdn_count)
+        oc_obj.optimize()
-    sys.exit(ret_code)
+        write_cfg_obj = WriteConfigFiles()
+        write_cfg_obj.write_dnsmasq_config(oc_obj.fqdn_dict)
+        write_cfg_obj.write_ipset_config(oc_obj.ip_dict, oc_obj.cidr_set)
+        write_cfg_obj.write_update_status_file(oc_obj.ip_count, oc_obj.cidr_count, oc_obj.output_fqdn_count)
+    sys.exit(1 if 1 in ret_list else (2 if 2 in ret_list else 0))

ruantiblock/Makefile

@@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=ruantiblock
 PKG_VERSION:=1.4
-PKG_RELEASE:=1
+PKG_RELEASE:=2
 PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
 
 include $(INCLUDE_DIR)/package.mk

ruantiblock/files/etc/ruantiblock/ruantiblock.conf

@@ -149,6 +149,7 @@ BLLIST_ALT_DNS_ADDR="8.8.8.8"
 ### Источники блэклиста
 RBL_ALL_URL="https://reestr.rublacklist.net/api/v3/snapshot/"
 RBL_IP_URL="https://reestr.rublacklist.net/api/v3/ips/"
+RBL_DPI_URL="https://reestr.rublacklist.net/api/v3/dpi/"
 ZI_ALL_URL="https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv"
 #ZI_ALL_URL="https://app.assembla.com/spaces/z-i/git/source/master/dump.csv?_format=raw"
 AF_IP_URL="https://antifilter.download/list/allyouneed.lst"

ruantiblock/files/usr/bin/ruantiblock

@@ -3,7 +3,7 @@
 ########################################################################
 #
 # Ruantiblock
-# (с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
+# (с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt)
 #
 ########################################################################
 
@@ -166,6 +166,7 @@ export BLLIST_ALT_DNS_ADDR="8.8.8.8"
 ### Источники блэклиста
 export RBL_ALL_URL="https://reestr.rublacklist.net/api/v3/snapshot/"
 export RBL_IP_URL="https://reestr.rublacklist.net/api/v3/ips/"
+export RBL_DPI_URL="https://reestr.rublacklist.net/api/v3/dpi/"
 export ZI_ALL_URL="https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv"
 export AF_IP_URL="https://antifilter.download/list/allyouneed.lst"
 export AF_FQDN_URL="https://antifilter.download/list/domains.lst"