diff --git a/autoinstall/current/autoinstall.sh b/autoinstall/current/autoinstall.sh index 5818e46..f2f635f 100755 --- a/autoinstall/current/autoinstall.sh +++ b/autoinstall/current/autoinstall.sh @@ -9,8 +9,8 @@ LUA_MODULE=0 LUCI_APP=1 OWRT_VERSION="current" -RUAB_VERSION="1.4-1" -RUAB_MOD_LUA_VERSION="1.4-0" +RUAB_VERSION="1.4-2" +RUAB_MOD_LUA_VERSION="1.4-2" RUAB_LUCI_APP_VERSION="1.4-2" BASE_URL="https://raw.githubusercontent.com/gSpotx2f/packages-openwrt/master" PKG_DIR="/tmp" diff --git a/ruantiblock-mod-lua/Makefile b/ruantiblock-mod-lua/Makefile index 759841a..b361c9d 100644 --- a/ruantiblock-mod-lua/Makefile +++ b/ruantiblock-mod-lua/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ruantiblock-mod-lua PKG_VERSION:=1.4 -PKG_RELEASE:=0 +PKG_RELEASE:=2 PKG_MAINTAINER:=gSpot include $(INCLUDE_DIR)/package.mk diff --git a/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua b/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua index ebd0f76..59d4ec9 100755 --- a/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua +++ b/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua @@ -1,7 +1,7 @@ #!/usr/bin/env lua --[[ - (с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) + (с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) lua == 5.1 --]] @@ -66,6 +66,7 @@ local Config = Class(nil, { ["UPDATE_STATUS_FILE"] = true, ["RBL_ALL_URL"] = true, ["RBL_IP_URL"] = true, + ["RBL_DPI_URL"] = true, ["ZI_ALL_URL"] = true, ["AF_IP_URL"] = true, ["AF_FQDN_URL"] = true, @@ -92,7 +93,7 @@ local Config = Class(nil, { encoding = "UTF-8", site_encoding = "", http_send_headers = { - --["User-Agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0", + ["User-Agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0", }, }) Config.wget_user_agent = (Config.http_send_headers["User-Agent"]) and ' -U "' .. Config.http_send_headers["User-Agent"] .. '"' or '' @@ -211,7 +212,180 @@ else error("Config.ICONV_TYPE should be either 'lua' or 'standalone'") end ------------------------------ Summarize ------------------------------ +------------------------------ Classes ------------------------------- + +local BlackListParser = Class(Config, { + ip_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?", + cidr_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?/%d%d?", + fqdn_pattern = "[a-z0-9_%.%-]-[a-z0-9_%-]+%.[a-z0-9%.%-]+", + url = "http://127.0.0.1", + records_separator = "\n", +}) + +function BlackListParser:new(t) + -- extended instance constructor + local instance = self(t) + instance.url = instance["url"] or self.url + instance.records_separator = instance["records_separator"] or self.records_separator + instance.site_encoding = instance["site_encoding"] or self.site_encoding + instance.cidr_count = 0 + instance.cidr_table = {} + instance.ip_subnet_table = {} + instance.ip_records_count = 0 + instance.ip_count = 0 + instance.ip_table = {} + instance.sld_table = {} + instance.fqdn_count = 0 + instance.fqdn_records_count = 0 + instance.fqdn_table = {} + instance.iconv_handler = iconv and iconv.open(instance.encoding, instance.site_encoding) or nil + return instance +end + +function BlackListParser:convert_encoding(input) + local output + if self.ICONV_TYPE == "lua" and self.iconv_handler then + output = self.iconv_handler:iconv(input) + elseif self.ICONV_TYPE == "standalone" and self.ICONV_CMD then + local iconv_handler = assert(io.popen('printf \'' .. input .. '\' | ' .. self.ICONV_CMD .. ' -f "' .. self.site_encoding .. '" -t "' .. self.encoding .. '"', 'r')) + output = iconv_handler:read("*a") + iconv_handler:close() + end + return (output) +end + +function BlackListParser:convert_to_punycode(input) + if self.site_encoding and self.site_encoding ~= "" then + input = self:convert_encoding(input) + end + return input and (idn.encode(input)) +end + +function BlackListParser:check_filter(str, filter_patterns, reverse) + if filter_patterns and str then + for pattern in pairs(filter_patterns) do + if str:match(pattern) then + return not reverse + end + end + end + return reverse +end + +function BlackListParser:get_subnet(ip) + return ip:match("^(%d+%.%d+%.%d+%.)%d+$") +end + +function BlackListParser:ip_value_processing(value) + if value and value ~= "" then + for ip_entry in value:gmatch(self.ip_pattern .. "/?%d?%d?") do + if not self.BLLIST_IP_FILTER or (self.BLLIST_IP_FILTER and not self:check_filter(ip_entry, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE)) then + if ip_entry:match("^" .. self.ip_pattern .. "$") and not self.ip_table[ip_entry] then + local subnet = self:get_subnet(ip_entry) + if subnet and (self.BLLIST_GR_EXCLUDED_NETS[subnet] or ((not self.BLLIST_IP_LIMIT or self.BLLIST_IP_LIMIT == 0) or (not self.ip_subnet_table[subnet] or self.ip_subnet_table[subnet] <= self.BLLIST_IP_LIMIT))) then + self.ip_table[ip_entry] = subnet + self.ip_subnet_table[subnet] = (self.ip_subnet_table[subnet] or 0) + 1 + self.ip_count = self.ip_count + 1 + end + elseif ip_entry:match("^" .. self.cidr_pattern .. "$") and not self.cidr_table[ip_entry] then + self.cidr_table[ip_entry] = true + self.cidr_count = self.cidr_count + 1 + end + end + end + end +end + +function BlackListParser:get_sld(fqdn) + return fqdn:match("^[a-z0-9_%.%-]-([a-z0-9_%-]+%.[a-z0-9%-]+)$") +end + +function BlackListParser:fqdn_value_processing(value) + value = value:gsub("%*%.", ""):gsub("%.$", ""):lower() + if self.BLLIST_STRIP_WWW then + value = value:gsub("^www[0-9]?%.", "") + end + if not self.BLLIST_FQDN_FILTER or (self.BLLIST_FQDN_FILTER and not self:check_filter(value, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)) then + if value:match("^" .. self.fqdn_pattern .. "$") then + elseif self.BLLIST_ENABLE_IDN and value:match("^[^\\/&%?]-[^\\/&%?%.]+%.[^\\/&%?%.]+%.?$") then + value = self:convert_to_punycode(value) + if not value then + return false + end + else + return false + end + local sld = self:get_sld(value) + if sld and (self.BLLIST_GR_EXCLUDED_SLD[sld] or ((not self.BLLIST_SD_LIMIT or self.BLLIST_SD_LIMIT == 0) or (not self.sld_table[sld] or self.sld_table[sld] < self.BLLIST_SD_LIMIT))) then + self.fqdn_table[value] = sld + self.sld_table[sld] = (self.sld_table[sld] or 0) + 1 + self.fqdn_count = self.fqdn_count + 1 + end + end + return true +end + +function BlackListParser:parser_func() + -- Must be overridden by a subclass + error("Method BlackListParser:parser_func() must be overridden by a subclass!") +end + +function BlackListParser:chunk_buffer() + local buff = "" + local ret_value = "" + local last_chunk + return function(chunk) + if last_chunk then + return nil + end + if chunk then + buff = buff .. chunk + local last_rs_position = select(2, buff:find("^.*" .. self.records_separator)) + if last_rs_position then + ret_value = buff:sub(1, last_rs_position) + buff = buff:sub((last_rs_position + 1), -1) + else + ret_value = "" + end + else + ret_value = buff + last_chunk = true + end + return (ret_value) + end +end + +function BlackListParser:get_http_data(url) + local ret_val, ret_code, ret_headers + local http_module = url:match("^https") and https or http + if http_module then + local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func()) + ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers} + if not ret_val or ret_code ~= 200 then + ret_val = nil + print(string.format("Connection error! (%s) URL: %s", ret_code, url)) + end + else + local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func()) + ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink) + end + return (ret_val == 1) and true or false +end + +function BlackListParser:run() + local return_code = 0 + if self:get_http_data(self.url) then + if (self.fqdn_count + self.ip_count + self.cidr_count) > self.BLLIST_MIN_ENTRIES then + return_code = 0 + else + return_code = 2 + end + else + return_code = 1 + end + return return_code +end + local Summarize = { HOSTS_LIMIT = 0, @@ -407,127 +581,27 @@ function Summarize:summarize_nets(cidr_list, modify_raw_list) ) end ------------------------------- Classes ------------------------------- -local BlackListParser = Class(Config, { - ip_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?", - cidr_pattern = "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?/%d%d?", - fqdn_pattern = "[a-z0-9_%.%-]-[a-z0-9_%-]+%.[a-z0-9%.%-]", - url = "http://127.0.0.1", - records_separator = "\n", - ips_separator = "|", +local OptimizeConfig = Class(Config, { + parsers_list = {}, }) -function BlackListParser:new(t) +function OptimizeConfig:new(t) -- extended instance constructor local instance = self(t) - instance.url = instance["url"] or self.url - instance.records_separator = instance["records_separator"] or self.records_separator - instance.ips_separator = instance["ips_separator"] or self.ips_separator - instance.site_encoding = instance["site_encoding"] or self.site_encoding - instance.ip_subnet_table = {} + instance.parsers_list = instance.parsers_list or self.parsers_list instance.cidr_count = 0 + instance.cidr_table = {} + instance.ip_subnet_table = {} instance.ip_records_count = 0 - instance.ip_count = 0 - instance.fqdn_count = 0 + instance.ip_table = {} instance.sld_table = {} instance.fqdn_records_count = 0 - instance.cidr_table = {} - instance.ip_table = {} instance.fqdn_table = {} - instance.iconv_handler = iconv and iconv.open(instance.encoding, instance.site_encoding) or nil return instance end -function BlackListParser:convert_encoding(input) - local output - if self.ICONV_TYPE == "lua" and self.iconv_handler then - output = self.iconv_handler:iconv(input) - elseif self.ICONV_TYPE == "standalone" and self.ICONV_CMD then - local iconv_handler = assert(io.popen('printf \'' .. input .. '\' | ' .. self.ICONV_CMD .. ' -f "' .. self.site_encoding .. '" -t "' .. self.encoding .. '"', 'r')) - output = iconv_handler:read("*a") - iconv_handler:close() - end - return (output) -end - -function BlackListParser:convert_to_punycode(input) - if self.site_encoding and self.site_encoding ~= "" then - input = self:convert_encoding(input) - end - return input and (idn.encode(input)) -end - -function BlackListParser:check_filter(str, filter_patterns, reverse) - if filter_patterns and str then - for pattern in pairs(filter_patterns) do - if str:match(pattern) then - return not reverse - end - end - end - return reverse -end - -function BlackListParser:get_subnet(ip) - return ip:match("^(%d+%.%d+%.%d+%.)%d+$") -end - -function BlackListParser:fill_ip_tables(val) - if val and val ~= "" then - for ip_entry in val:gmatch(self.ip_pattern .. "/?%d?%d?") do - if not self.BLLIST_IP_FILTER or (self.BLLIST_IP_FILTER and not self:check_filter(ip_entry, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE)) then - if ip_entry:match("^" .. self.ip_pattern .. "$") and not self.ip_table[ip_entry] then - local subnet = self:get_subnet(ip_entry) - if subnet and (self.BLLIST_GR_EXCLUDED_NETS[subnet] or ((not self.BLLIST_IP_LIMIT or self.BLLIST_IP_LIMIT == 0) or (not self.ip_subnet_table[subnet] or self.ip_subnet_table[subnet] <= self.BLLIST_IP_LIMIT))) then - self.ip_table[ip_entry] = subnet - self.ip_subnet_table[subnet] = (self.ip_subnet_table[subnet] or 0) + 1 - self.ip_count = self.ip_count + 1 - end - elseif ip_entry:match("^" .. self.cidr_pattern .. "$") and not self.cidr_table[ip_entry] then - self.cidr_table[ip_entry] = true - self.cidr_count = self.cidr_count + 1 - end - end - end - end -end - -function BlackListParser:get_sld(fqdn) - return fqdn:match("^[a-z0-9_%.%-]-([a-z0-9_%-]+%.[a-z0-9%-]+)$") -end - -function BlackListParser:fill_domain_tables(val) - val = val:gsub("%*%.", ""):gsub("%.$", ""):lower() - if self.BLLIST_STRIP_WWW then - val = val:gsub("^www[0-9]?%.", "") - end - if not self.BLLIST_FQDN_FILTER or (self.BLLIST_FQDN_FILTER and not self:check_filter(val, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE)) then - if val:match("^" .. self.fqdn_pattern .. "+$") then - elseif self.BLLIST_ENABLE_IDN and val:match("^[^\\/&%?]-[^\\/&%?%.]+%.[^\\/&%?%.]+%.?$") then - val = self:convert_to_punycode(val) - if not val then - return false - end - else - return false - end - local sld = self:get_sld(val) - if sld and (self.BLLIST_GR_EXCLUDED_SLD[sld] or ((not self.BLLIST_SD_LIMIT or self.BLLIST_SD_LIMIT == 0) or (not self.sld_table[sld] or self.sld_table[sld] < self.BLLIST_SD_LIMIT))) then - self.fqdn_table[val] = sld - self.sld_table[sld] = (self.sld_table[sld] or 0) + 1 - self.fqdn_count = self.fqdn_count + 1 - end - end - return true -end - -function BlackListParser:sink() - -- Must be overridden by a subclass - error("Method BlackListParser:sink() must be overridden by a subclass!") -end - -function BlackListParser:optimize_ip_table() +function OptimizeConfig:_optimize_ip_table() local optimized_table = {} for ipaddr, subnet in pairs(self.ip_table) do if self.ip_subnet_table[subnet] then @@ -544,8 +618,7 @@ function BlackListParser:optimize_ip_table() self.ip_table = optimized_table end - -function BlackListParser:optimize_fqdn_table() +function OptimizeConfig:_optimize_fqdn_table() local optimized_table = {} if self.BLLIST_GR_EXCLUDED_MASKS and #self.BLLIST_GR_EXCLUDED_MASKS > 0 then for sld in pairs(self.sld_table) do @@ -571,19 +644,60 @@ function BlackListParser:optimize_fqdn_table() self.fqdn_table = optimized_table end -function BlackListParser:group_ip_ranges() +function OptimizeConfig:_group_ip_ranges() for i in Summarize:summarize_ip_ranges(self.ip_table, true) do self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true end end -function BlackListParser:group_cidr_ranges() +function OptimizeConfig:_group_cidr_ranges() for i in Summarize:summarize_nets(self.cidr_table, true) do self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true end end -function BlackListParser:write_ipset_config() +function OptimizeConfig:_union(t1, t2) + local new_items = 0 + for k, v in pairs(t2) do + if t1[k] == nil then + t1[k] = v + new_items = new_items + 1 + end + end + return new_items +end + +function OptimizeConfig:optimize() + for _, i in ipairs(self.parsers_list) do + self:_union(self.cidr_table, i.cidr_table) + self:_union(self.ip_table, i.ip_table) + self:_union(self.ip_subnet_table, i.ip_subnet_table) + self:_union(self.fqdn_table, i.fqdn_table) + self:_union(self.sld_table, i.sld_table) + end + self:_optimize_fqdn_table() + self:_optimize_ip_table() + if self.BLLIST_SUMMARIZE_IP then + self:_group_ip_ranges() + end + if self.BLLIST_SUMMARIZE_CIDR then + self:_group_cidr_ranges() + end +end + + +local WriteConfigFiles = Class(Config, {}) + +function WriteConfigFiles:new(t) + -- extended instance constructor + local instance = self(t) + instance.cidr_count = 0 + instance.ip_count = 0 + instance.fqdn_count = 0 + return instance +end + +function WriteConfigFiles:write_ipset_config(ip_table, cidr_table) local file_handler = assert(io.open(self.IP_DATA_FILE, "w"), "Could not open nftset config") for _, v in ipairs({ self.NFTSET_CIDR, self.NFTSET_IP }) do file_handler:write(string.format("flush set %s %s\n", self.NFT_TABLE, v)) @@ -592,9 +706,9 @@ function BlackListParser:write_ipset_config() string.format("table %s {\n%s", self.NFT_TABLE, self.NFTSET_CIDR_CFG) ) local c = 0 - if next(self.cidr_table) then + if next(cidr_table) then file_handler:write("elements={") - for cidr in pairs(self.cidr_table) do + for cidr in pairs(cidr_table) do file_handler:write(string.format("%s,", cidr)) c = c + 1 end @@ -605,123 +719,59 @@ function BlackListParser:write_ipset_config() string.format("}\n%s", self.NFTSET_IP_CFG) ) local i = 0 - if next(self.ip_table) then + if next(ip_table) then file_handler:write("elements={") - for ipaddr in pairs(self.ip_table) do + for ipaddr in pairs(ip_table) do file_handler:write(string.format("%s,", ipaddr)) i = i + 1 end file_handler:write("};") end - self.ip_records_count = i + self.ip_count = i file_handler:write("}\n}\n") file_handler:close() end -function BlackListParser:write_dnsmasq_config() +function WriteConfigFiles:write_dnsmasq_config(fqdn_table) local file_handler = assert(io.open(self.DNSMASQ_DATA_FILE, "w"), "Could not open dnsmasq config") - for fqdn in pairs(self.fqdn_table) do + local i = 0 + for fqdn in pairs(fqdn_table) do if self.BLLIST_ALT_NSLOOKUP then file_handler:write(string.format("server=/%s/%s\n", fqdn, self.BLLIST_ALT_DNS_ADDR)) end file_handler:write(string.format("nftset=/%s/%s#%s\n", fqdn, self.NFT_TABLE_DNSMASQ, self.NFTSET_DNSMASQ)) + i = i + 1 end + self.fqdn_count = i file_handler:close() end -function BlackListParser:write_update_status() +function WriteConfigFiles:write_update_status_file() local file_handler = assert(io.open(self.UPDATE_STATUS_FILE, "w"), "Could not open 'update_status' file") - file_handler:write(string.format("%d %d %d", self.cidr_count, self.ip_records_count, self.fqdn_records_count)) + file_handler:write(string.format("%d %d %d", self.cidr_count, self.ip_count, self.fqdn_count)) file_handler:close() end -function BlackListParser:chunk_buffer() - local buff = "" - local ret_value = "" - local last_chunk - return function(chunk) - if last_chunk then - return nil - end - if chunk then - buff = buff .. chunk - local last_rs_position = select(2, buff:find("^.*" .. self.records_separator)) - if last_rs_position then - ret_value = buff:sub(1, last_rs_position) - buff = buff:sub((last_rs_position + 1), -1) - else - ret_value = "" - end - else - ret_value = buff - last_chunk = true - end - return (ret_value) - end -end +-- Parser subclasses -function BlackListParser:get_http_data(url) - local ret_val, ret_code, ret_headers - local http_module = url:match("^https") and https or http - if http_module then - local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:sink()) - ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers} - if not ret_val or ret_code ~= 200 then - ret_val = nil - print(string.format("Connection error! (%s) URL: %s", ret_code, url)) - end - else - local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:sink()) - ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink) - end - return (ret_val == 1) and true or false -end - -function BlackListParser:run() - local return_code = 0 - if self:get_http_data(self.url) then - if (self.fqdn_count + self.ip_count + self.cidr_count) > self.BLLIST_MIN_ENTRIES then - self:optimize_fqdn_table() - self:optimize_ip_table() - if self.BLLIST_SUMMARIZE_IP then - self:group_ip_ranges() - end - if self.BLLIST_SUMMARIZE_CIDR then - self:group_cidr_ranges() - end - self:write_ipset_config() - self:write_dnsmasq_config() - return_code = 0 - else - return_code = 2 - end - else - return_code = 1 - end - self:write_update_status() - return return_code -end - --- Subclasses - -local function ip_sink(self) +local function ip_parser_func(self) return function(chunk) if chunk and chunk ~= "" then for ip_string in chunk:gmatch(self.ip_string_pattern) do - self:fill_ip_tables(ip_string) + self:ip_value_processing(ip_string) end end return true end end -local function fqdn_sink_func(self, ip_str, fqdn_str) +local function fqdn_parser_func(self, ip_str, fqdn_str) if #fqdn_str > 0 and not fqdn_str:match("^" .. self.ip_pattern .. "$") then - if self:fill_domain_tables(fqdn_str) then + if self:fqdn_value_processing(fqdn_str) then return true end end - self:fill_ip_tables(ip_str) + self:ip_value_processing(ip_str) end -- rublacklist.net @@ -729,14 +779,13 @@ end local Rbl = Class(BlackListParser, { url = Config.RBL_ALL_URL, records_separator = '%{"appearDate": ', - ips_separator = ", ", }) -function Rbl:sink() +function Rbl:parser_func() return function(chunk) if chunk and chunk ~= "" then for fqdn_str, ip_str in chunk:gmatch('"domains": %["?(.-)"?%].-"ips": %[([a-f0-9/.:", ]*)%].-') do - fqdn_sink_func(self, ip_str, fqdn_str) + fqdn_parser_func(self, ip_str, fqdn_str) end end return true @@ -747,9 +796,28 @@ local RblIp = Class(Rbl, { url = Config.RBL_IP_URL, records_separator = ",", ip_string_pattern = "([a-f0-9/.:]+)", - sink = ip_sink, + parser_func = ip_parser_func, }) +local RblDPI = Class(BlackListParser, { + url = Config.RBL_DPI_URL, + BLLIST_MIN_ENTRIES = 0, + records_separator = '%{"domains"', +}) + +function RblDPI:parser_func() + return function(chunk) + if chunk and chunk ~= "" then + for fqdn_list in chunk:gmatch(': %[(.-)%]') do + for fqdn_str in fqdn_list:gmatch(self.fqdn_pattern) do + self:fqdn_value_processing(fqdn_str) + end + end + end + return true + end +end + -- zapret-info local Zi = Class(BlackListParser, { @@ -757,11 +825,11 @@ local Zi = Class(BlackListParser, { site_encoding = Config.ZI_ENCODING, }) -function Zi:sink() +function Zi:parser_func() return function(chunk) if chunk and chunk ~= "" then for ip_str, fqdn_str in chunk:gmatch("([^;]-);([^;]-);.-" .. self.records_separator) do - fqdn_sink_func(self, ip_str, fqdn_str) + fqdn_parser_func(self, ip_str, fqdn_str) end end return true @@ -770,7 +838,7 @@ end local ZiIp = Class(Zi, { ip_string_pattern = "([a-f0-9%.:/ |]+);.-\n", - sink = ip_sink, + parser_func = ip_parser_func, }) -- antifilter @@ -779,12 +847,12 @@ local Af = Class(BlackListParser, { url = Config.AF_FQDN_URL, }) -function Af:sink() +function Af:parser_func() local entry_pattern = "((.-))" .. self.records_separator return function(chunk) if chunk and chunk ~= "" then for fqdn_str, ip_str in chunk:gmatch(entry_pattern) do - fqdn_sink_func(self, ip_str, fqdn_str) + fqdn_parser_func(self, ip_str, fqdn_str) end end return true @@ -794,7 +862,7 @@ end local AfIp = Class(Af, { url = Config.AF_IP_URL, ip_string_pattern = "(.-)\n", - sink = ip_sink, + parser_func = ip_parser_func, }) -- ruantiblock @@ -821,7 +889,7 @@ function Ra:chunk_buffer() end end -function Ra:sink() +function Ra:parser_func() return function(chunk) if chunk and chunk ~= "" then self.current_file_handler:write(chunk) @@ -850,17 +918,34 @@ local RaIp = Class(Ra, { ----------------------------- Main section ------------------------------ -local ctx_table = { - ["ip"] = {["rublacklist"] = RblIp, ["zapret-info"] = ZiIp, ["antifilter"] = AfIp, ["ruantiblock"] = RaIp}, - ["fqdn"] = {["rublacklist"] = Rbl, ["zapret-info"] = Zi, ["antifilter"] = Af, ["ruantiblock"] = Ra}, +local parsers_table = { + ["ip"] = {["rublacklist"] = {RblIp}, ["zapret-info"] = {ZiIp}, ["antifilter"] = {AfIp}, ["ruantiblock"] = {RaIp}}, + ["fqdn"] = {["rublacklist"] = {Rbl, RblDPI}, ["zapret-info"] = {Zi}, ["antifilter"] = {Af}, ["ruantiblock"] = {Ra}}, } -local return_code = 1 -local ctx = ctx_table[Config.BLLIST_MODE] and ctx_table[Config.BLLIST_MODE][Config.BLLIST_SOURCE] -if ctx then - return_code = ctx:new():run() +local ret_list = {} +local parser_classes = parsers_table[Config.BLLIST_MODE] and parsers_table[Config.BLLIST_MODE][Config.BLLIST_SOURCE] +if parser_classes then + local parser_instances = {} + for _, i in ipairs(parser_classes) do + parser_instances[#parser_instances + 1] = i:new() + end + for _, i in ipairs(parser_instances) do + ret_list[i:run()] = true + end + return_sum = 0 + for i, _ in pairs(ret_list) do + return_sum = return_sum + i + end + if return_sum == 0 and Config.BLLIST_SOURCE ~= "ruantiblock" then + local oc_obj = OptimizeConfig:new({parsers_list = parser_instances}) + oc_obj:optimize() + local write_cfg_obj = WriteConfigFiles:new() + write_cfg_obj:write_dnsmasq_config(oc_obj.fqdn_table) + write_cfg_obj:write_ipset_config(oc_obj.ip_table, oc_obj.cidr_table) + write_cfg_obj:write_update_status_file() + end else - error("Wrong configuration! (Config.BLLIST_MODE or Config.BLLIST_SOURCE)") + error("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)") end - -os.exit(return_code) +os.exit(ret_list[1] and 1 or (ret_list[2] and 2 or 0)) diff --git a/ruantiblock-mod-py/Makefile b/ruantiblock-mod-py/Makefile index 4c4b1aa..a420fe9 100644 --- a/ruantiblock-mod-py/Makefile +++ b/ruantiblock-mod-py/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ruantiblock-mod-py PKG_VERSION:=1.4 -PKG_RELEASE:=0 +PKG_RELEASE:=2 PKG_MAINTAINER:=gSpot include $(INCLUDE_DIR)/package.mk diff --git a/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py b/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py index 443c892..6f40fcd 100755 --- a/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py +++ b/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -(с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) +(с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) Python >= 3.6 """ @@ -50,6 +50,7 @@ class Config: "UPDATE_STATUS_FILE", "RBL_ALL_URL", "RBL_IP_URL", + "RBL_DPI_URL", "ZI_ALL_URL", "AF_IP_URL", "AF_FQDN_URL", @@ -120,98 +121,6 @@ class Config: cls._load_filter(file_path or cls.BLLIST_IP_FILTER_FILE, cls.BLLIST_IP_FILTER_PATTERNS) -class Summarize: - HOSTS_LIMIT = 0 - NETS_LIMIT = 0 - - @staticmethod - def _sort_ip_func(e): - return IPv4Address(e) - - @classmethod - def _group_ip_ranges(cls, ip_list, raw_list=None): - - def remove_items(start, end): - for ip in range(int(start), int(end) + 1): - raw_list.remove(str(IPv4Address(ip))) - - start = end = None - hosts = 1 - for ip in ip_list: - ip_obj = IPv4Address(ip) - if end and (end + 1) == ip_obj: - hosts += 1 - else: - if hosts > 1 and hosts >= cls.HOSTS_LIMIT: - if raw_list: - remove_items(start, end) - yield start, end - start = ip_obj - hosts = 1 - end = ip_obj - else: - if hosts > 1 and hosts >= HOSTS_LIMIT: - if raw_list: - remove_items(start, end) - yield start, end - - @classmethod - def summarize_ip_ranges(cls, ip_list, modify_raw_list=False): - for s, e in cls._group_ip_ranges(sorted(ip_list, key=cls._sort_ip_func), - modify_raw_list and ip_list): - for i in summarize_address_range(s, e): - if i.prefixlen == 32: - if modify_raw_list: - if type(ip_list) == set: - ip_list.add(i.network_address) - else: - ip_list.append(i.network_address) - else: - yield i - - @staticmethod - def _sort_net_func(e): - return IPv4Network(e) - - @classmethod - def _group_nets(cls, cidr_list, raw_list=None): - - def remove_items(start, end): - for ip in range(int(start), int(end) + 1, 256): - raw_list.remove(str(IPv4Address(ip)) + "/24") - - start = end = curr_super_net = None - nets = 1 - for net in cidr_list: - net_obj = IPv4Network(net) - prefix_len = net_obj.prefixlen - if prefix_len == 24: - address = net_obj.network_address - super_net = net_obj.supernet(new_prefix=16) - if end and super_net == curr_super_net and (end + 256) == address: - nets += 1 - else: - if nets > 1 and nets >= cls.NETS_LIMIT: - if raw_list: - remove_items(start, end) - yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) - start = address - curr_super_net = super_net - nets = 1 - end = address - else: - if nets > 1 and nets >= cls.NETS_LIMIT: - if raw_list: - remove_items(start, end) - yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) - - @classmethod - def summarize_nets(cls, cidr_list): - for i in cls._group_nets(sorted(cidr_list, key=cls._sort_net_func), cidr_list): - for j in i: - yield j - - class ParserError(Exception): def __init__(self, reason=None): super().__init__(reason) @@ -235,16 +144,16 @@ class BlackListParser(Config): self.www_pattern = re.compile(r"^www[0-9]?[.]") self.cyr_pattern = re.compile(r"[а-яё]", re.U) self.cidr_set = set() - self.ip_set = {} + self.ip_dict = {} self.ip_subnet_dict = {} - self.fqdn_set = {} + self.fqdn_dict = {} self.sld_dict = {} self.cidr_count = 0 self.ip_count = 0 self.output_fqdn_count = 0 self.ssl_unverified = False self.send_headers_dict = { - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0", + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0", } ### Proxies (ex.: self.proxies = {"http": "http://192.168.0.1:8080", "https": "http://192.168.0.1:8080"}) self.proxies = None @@ -252,8 +161,6 @@ class BlackListParser(Config): self.data_chunk = 2048 self.url = "http://127.0.0.1" self.records_separator = "\n" - self.fields_separator = ";" - self.ips_separator = "|" self.default_site_encoding = "utf-8" self.site_encoding = self.default_site_encoding @@ -343,22 +250,21 @@ class BlackListParser(Config): regexp_obj = self.ip_pattern.fullmatch(ip_addr) return regexp_obj.group(1) if regexp_obj else None - def ip_field_processing(self, string): - for i in string.split(self.ips_separator): - if self.BLLIST_IP_FILTER and self._check_filter( - i, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE): - continue - if self.ip_pattern.fullmatch(i) and i not in self.ip_set: - subnet = self._get_subnet(i) - if subnet in self.BLLIST_GR_EXCLUDED_NETS or ( - not self.BLLIST_IP_LIMIT or ( - subnet not in self.ip_subnet_dict or self.ip_subnet_dict[subnet] < self.BLLIST_IP_LIMIT - ) - ): - self.ip_set[i] = subnet - self.ip_subnet_dict[subnet] = (self.ip_subnet_dict.get(subnet) or 0) + 1 - elif self.cidr_pattern.fullmatch(i) and i not in self.cidr_set: - self.cidr_set.add(i) + def ip_value_processing(self, value): + if self.BLLIST_IP_FILTER and self._check_filter( + value, self.BLLIST_IP_FILTER_PATTERNS, self.BLLIST_IP_FILTER_TYPE): + return + if self.ip_pattern.fullmatch(value) and value not in self.ip_dict: + subnet = self._get_subnet(value) + if subnet in self.BLLIST_GR_EXCLUDED_NETS or ( + not self.BLLIST_IP_LIMIT or ( + subnet not in self.ip_subnet_dict or self.ip_subnet_dict[subnet] < self.BLLIST_IP_LIMIT + ) + ): + self.ip_dict[value] = subnet + self.ip_subnet_dict[subnet] = (self.ip_subnet_dict.get(subnet) or 0) + 1 + elif self.cidr_pattern.fullmatch(value) and value not in self.cidr_set: + self.cidr_set.add(value) def _convert_to_punycode(self, string): if self.cyr_pattern.search(string): @@ -376,26 +282,26 @@ class BlackListParser(Config): regexp_obj = self.fqdn_pattern.fullmatch(fqdn) return regexp_obj.group(2) if regexp_obj else None - def fqdn_field_processing(self, string): - if self.ip_pattern.fullmatch(string): + def fqdn_value_processing(self, value): + if self.ip_pattern.fullmatch(value): raise FieldValueError() - string = string.strip("*.").lower() + value = value.strip("*.").lower() if self.BLLIST_STRIP_WWW: - string = self.www_pattern.sub("", string) + value = self.www_pattern.sub("", value) if not self.BLLIST_FQDN_FILTER or ( self.BLLIST_FQDN_FILTER and not self._check_filter( - string, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE) + value, self.BLLIST_FQDN_FILTER_PATTERNS, self.BLLIST_FQDN_FILTER_TYPE) ): - if self.fqdn_pattern.fullmatch(string): - string = self._convert_to_punycode(string) - sld = self._get_sld(string) + if self.fqdn_pattern.fullmatch(value): + value = self._convert_to_punycode(value) + sld = self._get_sld(value) if sld in self.BLLIST_GR_EXCLUDED_SLD or ( not self.BLLIST_SD_LIMIT or ( sld not in self.sld_dict or self.sld_dict[sld] < self.BLLIST_SD_LIMIT ) ): self.sld_dict[sld] = (self.sld_dict.get(sld) or 0) + 1 - self.fqdn_set[string] = sld + self.fqdn_dict[value] = sld else: raise FieldValueError() @@ -403,47 +309,11 @@ class BlackListParser(Config): """Must be overridden by a subclass""" raise NotImplementedError() - def _check_sld_masks(self, sld): - if self.BLLIST_GR_EXCLUDED_MASKS: - for pattern in self.BLLIST_GR_EXCLUDED_MASKS: - if re.fullmatch(pattern, sld): - return True - return False - - def _optimize_fqdn_set(self): - optimized_set = set() - for fqdn, sld in self.fqdn_set.items(): - if sld and (fqdn == sld or sld not in self.fqdn_set) and self.sld_dict.get(sld): - if (not self._check_sld_masks(sld) and ( - self.BLLIST_SD_LIMIT and sld not in self.BLLIST_GR_EXCLUDED_SLD - )) and (self.sld_dict[sld] >= self.BLLIST_SD_LIMIT): - record_value = sld - del(self.sld_dict[sld]) - else: - record_value = fqdn - optimized_set.add(record_value) - self.output_fqdn_count += 1 - self.fqdn_set = optimized_set - - def _optimize_ip_set(self): - optimized_set = set() - for ip_addr, subnet in self.ip_set.items(): - if subnet in self.ip_subnet_dict: - if subnet not in self.BLLIST_GR_EXCLUDED_NETS and ( - self.BLLIST_IP_LIMIT and self.ip_subnet_dict[subnet] >= self.BLLIST_IP_LIMIT - ): - self.cidr_set.add(f"{subnet}0/24") - del(self.ip_subnet_dict[subnet]) - else: - optimized_set.add(ip_addr) - self.ip_count += 1 - self.ip_set = optimized_set - def _group_ip_ranges(self): if self.BLLIST_SUMMARIZE_IP: - for i in Summarize.summarize_ip_ranges(self.ip_set, True): + for i in Summarize.summarize_ip_ranges(self.ip_dict, True): self.cidr_set.add(i.with_prefixlen) - self.ip_count = len(self.ip_set) + self.ip_count = len(self.ip_dict) def _group_cidr_ranges(self): if self.BLLIST_SUMMARIZE_CIDR: @@ -457,17 +327,216 @@ class BlackListParser(Config): self.BLLIST_IP_FILTER_PATTERNS = self._compile_filter_patterns(self.BLLIST_IP_FILTER_PATTERNS) self.records_separator = bytes(self.records_separator, "utf-8") self.parser_func() - if (len(self.ip_set) + len(self.cidr_set) + len(self.fqdn_set)) >= self.BLLIST_MIN_ENTRIES: - self._optimize_fqdn_set() - self._optimize_ip_set() - self._group_ip_ranges() - self._group_cidr_ranges() + if (len(self.ip_dict) + len(self.cidr_set) + len(self.fqdn_dict)) >= self.BLLIST_MIN_ENTRIES: ret_value = 0 else: ret_value = 2 return ret_value +class Summarize: + HOSTS_LIMIT = 0 + NETS_LIMIT = 0 + + @staticmethod + def _sort_ip_func(e): + return IPv4Address(e) + + @classmethod + def _group_ip_ranges(cls, ip_list, raw_list=None): + def remove_items(start, end): + for ip in range(int(start), int(end) + 1): + raw_list.remove(str(IPv4Address(ip))) + + start = end = None + hosts = 1 + for ip in ip_list: + ip_obj = IPv4Address(ip) + if end and (end + 1) == ip_obj: + hosts += 1 + else: + if hosts > 1 and hosts >= cls.HOSTS_LIMIT: + if raw_list: + remove_items(start, end) + yield start, end + start = ip_obj + hosts = 1 + end = ip_obj + else: + if hosts > 1 and hosts >= HOSTS_LIMIT: + if raw_list: + remove_items(start, end) + yield start, end + + @classmethod + def summarize_ip_ranges(cls, ip_list, modify_raw_list=False): + for s, e in cls._group_ip_ranges(sorted(ip_list, key=cls._sort_ip_func), + modify_raw_list and ip_list): + for i in summarize_address_range(s, e): + if i.prefixlen == 32: + if modify_raw_list: + if type(ip_list) == set: + ip_list.add(i.network_address) + else: + ip_list.append(i.network_address) + else: + yield i + + @staticmethod + def _sort_net_func(e): + return IPv4Network(e) + + @classmethod + def _group_nets(cls, cidr_list, raw_list=None): + def remove_items(start, end): + for ip in range(int(start), int(end) + 1, 256): + raw_list.remove(str(IPv4Address(ip)) + "/24") + + start = end = curr_super_net = None + nets = 1 + for net in cidr_list: + net_obj = IPv4Network(net) + prefix_len = net_obj.prefixlen + if prefix_len == 24: + address = net_obj.network_address + super_net = net_obj.supernet(new_prefix=16) + if end and super_net == curr_super_net and (end + 256) == address: + nets += 1 + else: + if nets > 1 and nets >= cls.NETS_LIMIT: + if raw_list: + remove_items(start, end) + yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) + start = address + curr_super_net = super_net + nets = 1 + end = address + else: + if nets > 1 and nets >= cls.NETS_LIMIT: + if raw_list: + remove_items(start, end) + yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) + + @classmethod + def summarize_nets(cls, cidr_list): + for i in cls._group_nets(sorted(cidr_list, key=cls._sort_net_func), cidr_list): + for j in i: + yield j + + +class OptimizeConfig(Config): + def __init__(self, parsers_list): + self.parsers_list = parsers_list + self.cidr_set = set() + self.ip_dict = {} + self.ip_subnet_dict = {} + self.fqdn_dict = {} + self.sld_dict = {} + self.cidr_count = 0 + self.ip_count = 0 + self.output_fqdn_count = 0 + + def _check_sld_masks(self, sld): + if self.BLLIST_GR_EXCLUDED_MASKS: + for pattern in self.BLLIST_GR_EXCLUDED_MASKS: + if re.fullmatch(pattern, sld): + return True + return False + + def _optimize_fqdn_dict(self): + optimized_set = set() + for fqdn, sld in self.fqdn_dict.items(): + if sld and (fqdn == sld or sld not in self.fqdn_dict) and self.sld_dict.get(sld): + if (not self._check_sld_masks(sld) and ( + self.BLLIST_SD_LIMIT and sld not in self.BLLIST_GR_EXCLUDED_SLD + )) and (self.sld_dict[sld] >= self.BLLIST_SD_LIMIT): + record_value = sld + del(self.sld_dict[sld]) + else: + record_value = fqdn + optimized_set.add(record_value) + self.output_fqdn_count += 1 + self.fqdn_dict = optimized_set + + def _optimize_ip_dict(self): + optimized_set = set() + for ip_addr, subnet in self.ip_dict.items(): + if subnet in self.ip_subnet_dict: + if subnet not in self.BLLIST_GR_EXCLUDED_NETS and ( + self.BLLIST_IP_LIMIT and self.ip_subnet_dict[subnet] >= self.BLLIST_IP_LIMIT + ): + self.cidr_set.add(f"{subnet}0/24") + del(self.ip_subnet_dict[subnet]) + else: + optimized_set.add(ip_addr) + self.ip_count += 1 + self.ip_dict = optimized_set + + def _group_ip_ranges(self): + if self.BLLIST_SUMMARIZE_IP: + for i in Summarize.summarize_ip_ranges(self.ip_dict, True): + self.cidr_set.add(i.with_prefixlen) + self.ip_count = len(self.ip_dict) + + def _group_cidr_ranges(self): + if self.BLLIST_SUMMARIZE_CIDR: + for i in Summarize.summarize_nets(self.cidr_set): + self.cidr_set.add(i.with_prefixlen) + self.cidr_count = len(self.cidr_set) + + def optimize(self): + for i in self.parsers_list: + self.cidr_set |= i.cidr_set + self.ip_dict.update(i.ip_dict) + self.ip_subnet_dict.update(i.ip_subnet_dict) + self.fqdn_dict.update(i.fqdn_dict) + self.sld_dict.update(i.sld_dict) + self._optimize_fqdn_dict() + self._optimize_ip_dict() + self._group_ip_ranges() + self._group_cidr_ranges() + + +class WriteConfigFiles(Config): + def __init__(self): + self.write_buffer = -1 + + def write_ipset_config(self, ip_dict, cidr_set): + with open(self.IP_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler: + for i in (self.NFTSET_CIDR, self.NFTSET_IP): + file_handler.write("flush set {} {}\n".format(self.NFT_TABLE, i)) + file_handler.write( + "table {} {{\n{}".format(self.NFT_TABLE, self.NFTSET_CIDR_CFG) + ) + if len(cidr_set) > 0: + file_handler.write("elements={") + for i in cidr_set: + file_handler.write(f"{i},") + file_handler.write("};") + file_handler.write( + "}}\n{}".format(self.NFTSET_IP_CFG) + ) + if len(ip_dict) > 0: + file_handler.write("elements={") + for i in ip_dict: + file_handler.write(f"{i},") + file_handler.write("};") + file_handler.write("}\n}\n") + + def write_dnsmasq_config(self, fqdn_dict): + with open(self.DNSMASQ_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler: + for fqdn in fqdn_dict: + file_handler.write( + f"server=/{fqdn}/{self.BLLIST_ALT_DNS_ADDR}\nnftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n" + if self.BLLIST_ALT_NSLOOKUP else + f"nftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n") + + def write_update_status_file(self, ip_count, cidr_count, fqdn_count): + with open(self.UPDATE_STATUS_FILE, "wt") as file_handler: + file_handler.write( + f"{cidr_count} {ip_count} {fqdn_count}") + + class RblFQDN(BlackListParser): def __init__(self): super().__init__() @@ -484,12 +553,33 @@ class RblFQDN(BlackListParser): fqdn_string = res.group(1) if fqdn_string: try: - self.fqdn_field_processing(fqdn_string) + self.fqdn_value_processing(fqdn_string) except FieldValueError: - self.ip_field_processing(ip_string) + for i in ip_string.split(self.ips_separator): + self.ip_value_processing(i) else: - self.ip_field_processing(ip_string) + for i in ip_string.split(self.ips_separator): + self.ip_value_processing(i) +class RblDPI(BlackListParser): + def __init__(self): + super().__init__() + self.url = self.RBL_DPI_URL + self.BLLIST_MIN_ENTRIES = 0 + self.records_separator = '{"domains"' + + def parser_func(self): + for entry in self._split_entries(): + res = re.search(r': \[(.*?)\]', entry) + if not res: + continue + fqdn_string = res.group(1) + if fqdn_string: + for i in fqdn_string.split(', "'): + try: + self.fqdn_value_processing(i.strip('"')) + except FieldValueError: + pass class RblIp(BlackListParser): def __init__(self): @@ -499,7 +589,7 @@ class RblIp(BlackListParser): def parser_func(self): for entry in self._split_entries(): - self.ip_field_processing(re.sub(r'[\[\]" ]', "", entry)) + self.ip_value_processing(re.sub(r'[\[\]" ]', "", entry)) class ZiFQDN(BlackListParser): @@ -507,6 +597,8 @@ class ZiFQDN(BlackListParser): super().__init__() self.url = self.ZI_ALL_URL self.site_encoding = self.ZI_ENCODING + self.fields_separator = ";" + self.ips_separator = "|" def parser_func(self): for entry in self._split_entries(): @@ -514,11 +606,13 @@ class ZiFQDN(BlackListParser): try: if entry_list[1]: try: - self.fqdn_field_processing(entry_list[1]) + self.fqdn_value_processing(entry_list[1]) except FieldValueError: - self.ip_field_processing(entry_list[0]) + for i in entry_list[0].split(self.ips_separator): + self.ip_value_processing(i) else: - self.ip_field_processing(entry_list[0]) + for i in entry_list[0].split(self.ips_separator): + self.ip_value_processing(i) except IndexError: pass @@ -527,7 +621,8 @@ class ZiIp(ZiFQDN): def parser_func(self): for entry in self._split_entries(): entry_list = entry.split(self.fields_separator) - self.ip_field_processing(entry_list[0]) + for i in entry_list[0].split(self.ips_separator): + self.ip_value_processing(i) class AfFQDN(BlackListParser): @@ -538,9 +633,9 @@ class AfFQDN(BlackListParser): def parser_func(self): for entry in self._split_entries(): try: - self.fqdn_field_processing(entry) + self.fqdn_value_processing(entry) except FieldValueError: - self.ip_field_processing(entry) + self.ip_value_processing(entry) class AfIp(BlackListParser): @@ -550,7 +645,7 @@ class AfIp(BlackListParser): def parser_func(self): for entry in self._split_entries(): - self.ip_field_processing(entry) + self.ip_value_processing(entry) class RaFQDN(BlackListParser): @@ -586,64 +681,27 @@ class RaIp(RaFQDN): self.url_stat = self.RA_IP_STAT_URL -class WriteConfigFiles(Config): - def __init__(self): - self.write_buffer = -1 - - def write_ipset_config(self, ip_set, cidr_set): - with open(self.IP_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler: - for i in (self.NFTSET_CIDR, self.NFTSET_IP): - file_handler.write("flush set {} {}\n".format(self.NFT_TABLE, i)) - file_handler.write( - "table {} {{\n{}".format(self.NFT_TABLE, self.NFTSET_CIDR_CFG) - ) - if len(cidr_set) > 0: - file_handler.write("elements={") - for i in cidr_set: - file_handler.write(f"{i},") - file_handler.write("};") - file_handler.write( - "}}\n{}".format(self.NFTSET_IP_CFG) - ) - if len(ip_set) > 0: - file_handler.write("elements={") - for i in ip_set: - file_handler.write(f"{i},") - file_handler.write("};") - file_handler.write("}\n}\n") - - def write_dnsmasq_config(self, fqdn_set): - with open(self.DNSMASQ_DATA_FILE, "wt", buffering=self.write_buffer) as file_handler: - for fqdn in fqdn_set: - file_handler.write( - f"server=/{fqdn}/{self.BLLIST_ALT_DNS_ADDR}\nnftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n" - if self.BLLIST_ALT_NSLOOKUP else - f"nftset=/{fqdn}/{self.NFT_TABLE_DNSMASQ}#{self.NFTSET_DNSMASQ}\n") - - def write_update_status_file(self, ip_count, cidr_count, output_fqdn_count): - with open(self.UPDATE_STATUS_FILE, "wt") as file_handler: - file_handler.write( - f"{cidr_count} {ip_count} {output_fqdn_count}") - - if __name__ == "__main__": Config.load_environ_config() Config.load_fqdn_filter() Config.load_ip_filter() - ctx_dict = { - "ip": {"rublacklist": RblIp, "zapret-info": ZiIp, "antifilter": AfIp, "ruantiblock": RaIp}, - "fqdn": {"rublacklist": RblFQDN, "zapret-info": ZiFQDN, "antifilter": AfFQDN, "ruantiblock": RaFQDN}, + parsers_dict = { + "ip": {"rublacklist": [RblIp], "zapret-info": [ZiIp], "antifilter": [AfIp], "ruantiblock": [RaIp]}, + "fqdn": {"rublacklist": [RblFQDN, RblDPI], "zapret-info": [ZiFQDN], "antifilter": [AfFQDN], "ruantiblock": [RaFQDN]}, } - write_cfg_obj = WriteConfigFiles() try: - ctx = ctx_dict[Config.BLLIST_MODE][Config.BLLIST_SOURCE]() + parser_classes = parsers_dict[Config.BLLIST_MODE][Config.BLLIST_SOURCE] except KeyError: - print("Wrong configuration! (Config.BLLIST_MODE or Config.BLLIST_SOURCE)", + print("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)", file=sys.stderr) sys.exit(1) - ret_code = ctx.run() - if ret_code == 0 and Config.BLLIST_SOURCE != "ruantiblock": - write_cfg_obj.write_dnsmasq_config(ctx.fqdn_set) - write_cfg_obj.write_ipset_config(ctx.ip_set, ctx.cidr_set) - write_cfg_obj.write_update_status_file(ctx.ip_count, ctx.cidr_count, ctx.output_fqdn_count) - sys.exit(ret_code) + parser_instances = [i() for i in parser_classes] + ret_list = [i.run() for i in parser_instances] + if sum(ret_list) == 0 and Config.BLLIST_SOURCE != "ruantiblock": + oc_obj = OptimizeConfig(parser_instances) + oc_obj.optimize() + write_cfg_obj = WriteConfigFiles() + write_cfg_obj.write_dnsmasq_config(oc_obj.fqdn_dict) + write_cfg_obj.write_ipset_config(oc_obj.ip_dict, oc_obj.cidr_set) + write_cfg_obj.write_update_status_file(oc_obj.ip_count, oc_obj.cidr_count, oc_obj.output_fqdn_count) + sys.exit(1 if 1 in ret_list else (2 if 2 in ret_list else 0)) diff --git a/ruantiblock/Makefile b/ruantiblock/Makefile index d5e0999..850be63 100644 --- a/ruantiblock/Makefile +++ b/ruantiblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ruantiblock PKG_VERSION:=1.4 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_MAINTAINER:=gSpot include $(INCLUDE_DIR)/package.mk diff --git a/ruantiblock/files/etc/ruantiblock/ruantiblock.conf b/ruantiblock/files/etc/ruantiblock/ruantiblock.conf index f93b525..0296646 100644 --- a/ruantiblock/files/etc/ruantiblock/ruantiblock.conf +++ b/ruantiblock/files/etc/ruantiblock/ruantiblock.conf @@ -149,6 +149,7 @@ BLLIST_ALT_DNS_ADDR="8.8.8.8" ### Источники блэклиста RBL_ALL_URL="https://reestr.rublacklist.net/api/v3/snapshot/" RBL_IP_URL="https://reestr.rublacklist.net/api/v3/ips/" +RBL_DPI_URL="https://reestr.rublacklist.net/api/v3/dpi/" ZI_ALL_URL="https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv" #ZI_ALL_URL="https://app.assembla.com/spaces/z-i/git/source/master/dump.csv?_format=raw" AF_IP_URL="https://antifilter.download/list/allyouneed.lst" diff --git a/ruantiblock/files/usr/bin/ruantiblock b/ruantiblock/files/usr/bin/ruantiblock index e931986..7ec2169 100755 --- a/ruantiblock/files/usr/bin/ruantiblock +++ b/ruantiblock/files/usr/bin/ruantiblock @@ -3,7 +3,7 @@ ######################################################################## # # Ruantiblock -# (с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) +# (с) 2024 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) # ######################################################################## @@ -166,6 +166,7 @@ export BLLIST_ALT_DNS_ADDR="8.8.8.8" ### Источники блэклиста export RBL_ALL_URL="https://reestr.rublacklist.net/api/v3/snapshot/" export RBL_IP_URL="https://reestr.rublacklist.net/api/v3/ips/" +export RBL_DPI_URL="https://reestr.rublacklist.net/api/v3/dpi/" export ZI_ALL_URL="https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv" export AF_IP_URL="https://antifilter.download/list/allyouneed.lst" export AF_FQDN_URL="https://antifilter.download/list/domains.lst"