i11th/ruantiblock_openwrt
1
0
Fork 0
mirror of https://github.com/gSpotx2f/ruantiblock_openwrt.git synced 2026-05-13 14:10:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

ruantiblock-mod-lua, ruantiblock-mod-py: subdomains optimization.

Browse Source
This commit is contained in:
gSpot
2025-01-21 23:33:51 +03:00
parent 50a1c68afd
commit 8eaf8e3fd8
9 changed files with 124 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
autoinstall/2.x/autoinstall.sh
+3 -3
View File
@@ -10,9 +10,9 @@ LUCI_APP=1
HTTPS_DNS_PROXY=1
OWRT_VERSION="current"
RUAB_VERSION="2.1.2-r1"
RUAB_MOD_LUA_VERSION="2.1.2-r1"
RUAB_LUCI_APP_VERSION="2.1.2-r1"
RUAB_VERSION="2.1.3-r1"
RUAB_MOD_LUA_VERSION="2.1.3-r1"
RUAB_LUCI_APP_VERSION="2.1.3-r1"
BASE_URL="https://raw.githubusercontent.com/gSpotx2f/packages-openwrt/master"
PKG_DIR="/tmp"
luci-app-ruantiblock/Makefile
+1 -1
View File
@@ -5,7 +5,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=luci-app-ruantiblock
PKG_VERSION:=2.1.2
PKG_VERSION:=2.1.3
PKG_RELEASE:=1
LUCI_TITLE:=LuCI support for ruantiblock
LUCI_DEPENDS:=+ruantiblock
luci-app-ruantiblock/htdocs/luci-static/resources/view/ruantiblock/settings.js
-7
View File
@@ -149,13 +149,6 @@ return view.extend({
'<br /><code>poker<br />[ck]?a[sz]ino?<br />[vw]ulkan<br />slots?</code>'
);
let user_entries_edit = new tools.fileEditDialog(
tools.userEntriesFile,
_('User entries'),
_('One entry (IP, CIDR or FQDN) per line. In the FQDN records, you can specify the DNS server for resolving this domain (separated by a space). You can also comment on lines (<code>#</code> is the first character of a line).<br />Examples:') +
'<br /><code>#comment<br />domain.net<br />sub.domain.com 8.8.8.8<br />sub.domain.com 8.8.8.8#53<br />74.125.131.19<br />74.125.0.0/16</code>'
);
let bypass_entries_edit = new tools.fileEditDialog(
tools.bypassEntriesFile,
_('Exclusion list'),
ruantiblock-mod-lua/Makefile
+2 -1
View File
@@ -5,7 +5,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=ruantiblock-mod-lua
PKG_VERSION:=2.1.2
PKG_VERSION:=2.1.3
PKG_RELEASE:=1
PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
@@ -18,6 +18,7 @@ define Package/$(PKG_NAME)
URL:=https://github.com/gSpotx2f/ruantiblock_openwrt
PKGARCH:=all
DEPENDS:=+ruantiblock +lua +luasocket +luasec +luabitop +iconv +idn
#DEPENDS:=+ruantiblock +lua +luasocket +luasec +luabitop +iconv +idn +lua-lzlib
endef
define Package/$(PKG_NAME)/description
ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua
+85 -16
View File
@@ -251,6 +251,12 @@ if not it then
Config.BLLIST_SUMMARIZE_CIDR = false
Config.BLLIST_SUMMARIZE_IP = false
end
--[[
local zlib = prequire("zlib")
if Config.BLLIST_ENABLE_IDN and not idn then
error("You need to install lua-lzlib...")
end
--]]
-- Iconv check
@@ -392,6 +398,12 @@ function BlackListParser:parser_func()
error("Method BlackListParser:parser_func() must be overridden by a subclass!")
end
function BlackListParser:prepare_data()
return function(chunk)
return chunk
end
end
function BlackListParser:chunk_buffer()
local ret_value = ""
local last_chunk
@@ -425,7 +437,7 @@ function BlackListParser:get_http_data(url)
end
ret_val, ret_code, ret_headers = http_module.request{url = url, method="HEAD", headers = self.http_send_headers}
if ret_val and ret_code == 200 then
local http_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
local http_sink = ltn12.sink.chain(self:prepare_data(), self:chunk_buffer(), self:parser_func())
ret_val, ret_code, ret_headers = http_module.request{url = url, sink = http_sink, headers = self.http_send_headers}
self.http_codes[ret_code] = true
if not ret_val or ret_code ~= 200 then
@@ -437,7 +449,7 @@ function BlackListParser:get_http_data(url)
print(string.format("Connection error! (%s) URL: %s", ret_code, url))
end
else
local wget_sink = ltn12.sink.chain(self:chunk_buffer(), self:parser_func())
local wget_sink = ltn12.sink.chain(self:prepare_data(), self:chunk_buffer(), self:parser_func())
ret_val = ltn12.pump.all(ltn12.source.file(io.popen(self.WGET_CMD .. self.wget_user_agent .. ' "' .. url .. '"', 'r')), wget_sink)
end
return (ret_val == 1) and true or false
@@ -695,21 +707,25 @@ function OptimizeConfig:new(t)
return instance
end
function OptimizeConfig:_optimize_ip_table()
local optimized_table = {}
for ipaddr, subnet in pairs(self.ip_table) do
if self.ip_subnet_table[subnet] then
if (self.BLLIST_IP_LIMIT and self.BLLIST_IP_LIMIT > 0 and not self.BLLIST_GR_EXCLUDED_NETS_PATTERNS[subnet]) and self.ip_subnet_table[subnet] >= self.BLLIST_IP_LIMIT then
self.cidr_table[string.format("%s0/24", subnet)] = true
self.ip_subnet_table[subnet] = nil
self.cidr_count = self.cidr_count + 1
else
optimized_table[ipaddr] = true
self.ip_records_count = self.ip_records_count + 1
function OptimizeConfig:_remove_subdomains()
local tld_table = {}
for fqdn, sld in pairs(self.fqdn_table) do
if not tld_table[sld] then
tld_table[sld] = {}
end
tld_table[sld][fqdn] = true
end
for _, v in pairs(tld_table) do
for i in pairs(v) do
if self.fqdn_table[i] then
for j in pairs(v) do
if (j ~= i) and j:find("." .. i, 1, true) then
self.fqdn_table[j] = nil
end
end
end
end
end
self.ip_table = optimized_table
end
function OptimizeConfig:_optimize_fqdn_table()
@@ -728,6 +744,23 @@ function OptimizeConfig:_optimize_fqdn_table()
self.fqdn_table = optimized_table
end
function OptimizeConfig:_optimize_ip_table()
local optimized_table = {}
for ipaddr, subnet in pairs(self.ip_table) do
if self.ip_subnet_table[subnet] then
if (self.BLLIST_IP_LIMIT and self.BLLIST_IP_LIMIT > 0 and not self.BLLIST_GR_EXCLUDED_NETS_PATTERNS[subnet]) and self.ip_subnet_table[subnet] >= self.BLLIST_IP_LIMIT then
self.cidr_table[string.format("%s0/24", subnet)] = true
self.ip_subnet_table[subnet] = nil
self.cidr_count = self.cidr_count + 1
else
optimized_table[ipaddr] = true
self.ip_records_count = self.ip_records_count + 1
end
end
end
self.ip_table = optimized_table
end
function OptimizeConfig:_group_ip_ranges()
for i in Summarize:summarize_ip_ranges(self.ip_table, true) do
self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true
@@ -759,6 +792,7 @@ function OptimizeConfig:optimize()
self:_union(self.fqdn_table, i.fqdn_table)
self:_union(self.sld_table, i.sld_table)
end
self:_remove_subdomains()
self:_optimize_fqdn_table()
self:_optimize_ip_table()
if self.BLLIST_SUMMARIZE_IP then
@@ -909,6 +943,43 @@ local Zi = Class(BlackListParser, {
site_encoding = Config.ZI_ENCODING,
})
-- for https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
-- lua-lzlib --
--[[
function Zi:prepare_data()
local curr_chunk
local stream = zlib.inflate({
read = function(self, consume)
return curr_chunk
end,
})
return function(chunk)
if chunk then
curr_chunk = chunk
return (stream:read(2048)) or ""
end
stream:close()
return nil
end
end
--]]
-- lua-zlib --
--[[
function Zi:prepare_data()
local stream = zlib.inflate()
return function(chunk)
if chunk then
local inflated, eos = stream(chunk)
if eos then
stream = zlib.inflate()
end
return inflated
end
return nil
end
end
--]]
function Zi:parser_func()
return function(chunk)
if chunk and chunk ~= "" then
@@ -1075,7 +1146,6 @@ if parser_classes then
for _, i in ipairs(parser_instances) do
ret_list[i:run()] = true
end
local return_sum = 0
for i, _ in pairs(ret_list) do
return_sum = return_sum + i
@@ -1091,5 +1161,4 @@ if parser_classes then
else
error("Wrong configuration! (Config.BLLIST_MODE, Config.BLLIST_SOURCE)")
end
os.exit(ret_list[1] and 1 or (ret_list[2] and 2 or 0))
ruantiblock-mod-py/Makefile
+1 -1
View File
@@ -5,7 +5,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=ruantiblock-mod-py
PKG_VERSION:=2.1.2
PKG_VERSION:=2.1.3
PKG_RELEASE:=1
PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py
+29 -3
View File
@@ -15,6 +15,7 @@ import socket
import ssl
import sys
from urllib import request
#import zlib
class Config:
@@ -281,8 +282,12 @@ class BlackListParser(Config):
if not chunk:
break
def _align_chunk(self, url):
def prepare_data(self, url):
for chunk in self._download_data(url):
yield chunk
def _align_chunk(self, url):
for chunk in self.prepare_data(url):
if chunk is None:
yield self.rest
continue
@@ -458,7 +463,6 @@ class Summarize:
@classmethod
def _group_nets(cls, cidr_list, raw_list=None):
def remove_items(start, end):
for ip in range(int(start), int(end) + 1, 256):
raw_list.remove(str(IPv4Address(ip)) + "/24")
@@ -507,6 +511,18 @@ class OptimizeConfig(Config):
self.ip_count = 0
self.output_fqdn_count = 0
def _remove_subdomains(self):
tld_dict = {}
for fqdn, sld in self.fqdn_dict.items():
tld_dict.setdefault(sld, [])
tld_dict[sld].append(fqdn)
for v in tld_dict.values():
for i in v:
if i in self.fqdn_dict:
for j in v:
if (j != i) and j.endswith("." + i):
self.fqdn_dict.pop(j, None)
def _optimize_fqdn_dict(self):
optimized_set = set()
for fqdn, sld in self.fqdn_dict.items():
@@ -555,6 +571,7 @@ class OptimizeConfig(Config):
self.ip_subnet_dict.update(i.ip_subnet_dict)
self.fqdn_dict.update(i.fqdn_dict)
self.sld_dict.update(i.sld_dict)
self._remove_subdomains()
self._optimize_fqdn_dict()
self._optimize_ip_dict()
self._group_ip_ranges()
@@ -666,6 +683,16 @@ class ZiFQDN(BlackListParser):
self.site_encoding = self.ZI_ENCODING
self.fields_separator = ";"
self.ips_separator = "|"
# self.decomp_obj = zlib.decompressobj(wbits=47)
# def prepare_data(self, url):
# """
# for https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
# """
# for chunk in self._download_data(url):
# if chunk:
# data = self.decomp_obj.decompress(chunk)
# yield data
def parser_func(self):
for url in self.url:
@@ -693,7 +720,6 @@ class ZiIp(ZiFQDN):
for i in entry_list[0].split(self.ips_separator):
self.ip_value_processing(i)
class AfFQDN(BlackListParser):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
ruantiblock/Makefile
+1 -1
View File
@@ -5,7 +5,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=ruantiblock
PKG_VERSION:=2.1.2
PKG_VERSION:=2.1.3
PKG_RELEASE:=1
PKG_MAINTAINER:=gSpot <https://github.com/gSpotx2f/ruantiblock_openwrt>
ruantiblock/files/usr/bin/ruantiblock
+2 -2
View File
@@ -73,7 +73,7 @@ export ENABLE_LOGGING=1
### Вывод дополнительных сообщений в лог (0 - выкл, 1, 2)
export DEBUG=0
### Файл для вывода некоторых отладочных сообщений
export DEBUG_FILE="/tmp/ruanliblock.debug"
export DEBUG_FILE="/tmp/${NAME}.debug"
### Html-страница с инфо о текущем статусе (0 - выкл, 1 - вкл) (не используется в OpenWrt)
export ENABLE_HTML_INFO=0
### Максимальное кол-во элементов списка nftables
@@ -998,7 +998,7 @@ GetMainInstanceEntries() {
}
GetBlacklistFiles() {
local _return_code=1 _user_entries_ret_code=1
local _return_code=0 _user_entries_ret_code=1
AddBypassEntries
GetMainInstanceEntries
case $? in