From 80c18d0c744ab6dd0774efb538915ed0be5efd95 Mon Sep 17 00:00:00 2001 From: gSpot Date: Fri, 8 Sep 2023 21:53:56 +0300 Subject: [PATCH] ruantiblock-mod-lua, ruantiblock-mod-py: removed ruab_sum_ip.lua, ruab_sum_ip.py --- ruantiblock-mod-lua/Makefile | 4 +- .../files/usr/lib/lua/ruab_sum_ip.lua | 207 ----------------- .../usr/libexec/ruantiblock/ruab_parser.lua | 209 +++++++++++++++++- ruantiblock-mod-py/Makefile | 4 +- .../files/usr/lib/python3.11/ruab_sum_ip.py | 93 -------- .../usr/libexec/ruantiblock/ruab_parser.py | 100 ++++++++- 6 files changed, 299 insertions(+), 318 deletions(-) delete mode 100644 ruantiblock-mod-lua/files/usr/lib/lua/ruab_sum_ip.lua delete mode 100644 ruantiblock-mod-py/files/usr/lib/python3.11/ruab_sum_ip.py diff --git a/ruantiblock-mod-lua/Makefile b/ruantiblock-mod-lua/Makefile index 088c192..fd142bf 100644 --- a/ruantiblock-mod-lua/Makefile +++ b/ruantiblock-mod-lua/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ruantiblock-mod-lua PKG_VERSION:=1.3 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_MAINTAINER:=gSpot include $(INCLUDE_DIR)/package.mk @@ -32,8 +32,6 @@ endef define Package/$(PKG_NAME)/install $(INSTALL_DIR) $(1)/usr/libexec/ruantiblock $(INSTALL_BIN) ./files/usr/libexec/ruantiblock/ruab_parser.lua $(1)/usr/libexec/ruantiblock/ruab_parser.lua - $(INSTALL_DIR) $(1)/usr/lib/lua - $(INSTALL_DATA) ./files/usr/lib/lua/ruab_sum_ip.lua $(1)/usr/lib/lua/ruab_sum_ip.lua $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_DATA) ./files/etc/uci-defaults/ruantiblock-mod-lua $(1)/etc/uci-defaults/ruantiblock-mod-lua endef diff --git a/ruantiblock-mod-lua/files/usr/lib/lua/ruab_sum_ip.lua b/ruantiblock-mod-lua/files/usr/lib/lua/ruab_sum_ip.lua deleted file mode 100644 index 145c07b..0000000 --- a/ruantiblock-mod-lua/files/usr/lib/lua/ruab_sum_ip.lua +++ /dev/null @@ -1,207 +0,0 @@ ---[[ - (с) 2020 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) ---]] - -local it = require("iptool") - -HOSTS_LIMIT = 0 -NETS_LIMIT = 0 - -local function sort_ip_list(t) - local t2 = {} - for k in pairs(t) do - t2[#t2 + 1] = k - end - table.sort(t2, function(a, b) return it.ip_to_int(a) < it.ip_to_int(b) end) - return t2 -end - -local function group_ip_ranges(ip_list, raw_list) - local function remove_items(start, stop) - for i = start, stop do - if raw_list[i] then - raw_list[i] = nil - return - end - local item = it.int_to_ip(i) - if raw_list[item] then - raw_list[it.int_to_ip(i)] = nil - end - end - end - - local start, stop, last_call - local hosts = 1 - local i = 0 - return function() - local ret_val - local ip - repeat - i = i + 1 - ip = ip_list[i] - if ip then - local ip_dec = it.ip_to_int(ip) - if stop and (stop + 1) == ip_dec then - hosts = hosts + 1 - else - if hosts > 1 and hosts >= HOSTS_LIMIT then - if raw_list then - remove_items(start, stop) - end - ret_val = {[1] = start, [2] = stop} - start = ip_dec - stop = ip_dec - hosts = 1 - break - end - start = ip_dec - end - stop = ip_dec - elseif not last_call then - if hosts > 1 and hosts >= HOSTS_LIMIT then - if raw_list then - remove_items(start, stop) - end - ret_val = {[1] = start, [2] = stop} - last_call = true - break - end - end - until not ip - return ret_val - end -end - -local function sort_net_list(t) - local t2 = {} - for k, v in pairs(t) do - local ip, pref = it.get_network_addr(k) - t2[#t2 + 1] = {[1] = ip, [2] = pref} - end - table.sort(t2, function(a, b) return a[1] < b[1] end) - return t2 -end - -local function group_nets(cidr_list, raw_list) - local function remove_items(start, stop) - for i = start, stop, 256 do - local item = it.int_to_ip(i) .. "/24" - if raw_list[item] then - raw_list[item] = nil - end - end - end - - local start, stop, last_call, curr_supernet - local nets = 1 - local i = 0 - return function() - local ret_val - local cidr - repeat - i = i + 1 - cidr = cidr_list[i] - if cidr then - local network_address, prefixlen - if type(cidr) == "string" then - network_address, prefixlen = it.get_network_addr(cidr) - elseif type(cidr) == "table" then - network_address, prefixlen = cidr[1], cidr[2] - end - if prefixlen == 24 then - local supernet = it.get_supernet({[1] = network_address, [2] = prefixlen}, 16) - if stop and supernet == curr_supernet and (stop + 256) == network_address then - nets = nets + 1 - else - if nets > 1 and nets >= NETS_LIMIT then - if raw_list then - remove_items(start, stop) - end - ret_val = {[1] = start, [2] = stop + 255} - start = network_address - stop = network_address - nets = 1 - curr_supernet = supernet - break - end - start = network_address - curr_supernet = supernet - end - stop = network_address - end - elseif not last_call then - if nets > 1 and nets >= NETS_LIMIT then - if raw_list then - remove_items(start, stop) - end - ret_val = {[1] = start, [2] = stop + 255} - last_call = true - break - end - end - until not cidr - return ret_val - end -end - -local function summarize_ranges(ip_iter) - local s_range_iter - return function() - -- lua >= 5.2 - --::check_prefix:: - if s_range_iter then - repeat - local ip_t = s_range_iter() - if ip_t then - return ip_t - end - until not ip_t - end - local ip_range = ip_iter() - if ip_range then - s_range_iter = it.summarize_address_range(ip_range[1], ip_range[2]) - if s_range_iter then - -- lua >= 5.2 - --goto check_prefix - -- lua < 5.2 - repeat - local ip_t = s_range_iter() - if ip_t then - return ip_t - end - until not ip_t - -- - end - else - return - end - end -end - -function summarize_ip_ranges(ip_list, modify_raw_list) - local summ_iter = summarize_ranges(group_ip_ranges(sort_ip_list(ip_list), - modify_raw_list and ip_list) - ) - return function() - repeat - local ip_t = summ_iter() - if ip_t and ip_t[2] == 32 then - if modify_raw_list then - ip_list[it.int_to_ip(ip_t[1])] = true - end - else - return ip_t - end - until not ip_t - end -end - -function summarize_nets(cidr_list, modify_raw_list) - return summarize_ranges(group_nets(sort_net_list(cidr_list), - modify_raw_list and cidr_list)) -end - -return { - summarize_ip_ranges = summarize_ip_ranges, - summarize_nets = summarize_nets, -} diff --git a/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua b/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua index 8f4343f..ebd0f76 100755 --- a/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua +++ b/ruantiblock-mod-lua/files/usr/libexec/ruantiblock/ruab_parser.lua @@ -1,7 +1,7 @@ #!/usr/bin/env lua --[[ - (с) 2020 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) + (с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) lua == 5.1 --]] @@ -188,14 +188,11 @@ if Config.BLLIST_ENABLE_IDN and not idn then end local iconv = prequire("iconv") -local si, it +local it if prequire("bit") then it = prequire("iptool") - if it then - si = prequire("ruab_sum_ip") - end end -if not si then +if not it then Config.BLLIST_SUMMARIZE_CIDR = false Config.BLLIST_SUMMARIZE_IP = false end @@ -214,6 +211,202 @@ else error("Config.ICONV_TYPE should be either 'lua' or 'standalone'") end +----------------------------- Summarize ------------------------------ + +local Summarize = { + HOSTS_LIMIT = 0, + NETS_LIMIT = 0, +} + +function Summarize:_sort_ip_list(t) + local t2 = {} + for k in pairs(t) do + t2[#t2 + 1] = k + end + table.sort(t2, function(a, b) return it.ip_to_int(a) < it.ip_to_int(b) end) + return t2 +end + +function Summarize:_group_ip_ranges(ip_list, raw_list) + local function remove_items(start, stop) + for i = start, stop do + if raw_list[i] then + raw_list[i] = nil + return + end + local item = it.int_to_ip(i) + if raw_list[item] then + raw_list[it.int_to_ip(i)] = nil + end + end + end + + local start, stop, last_call + local hosts = 1 + local i = 0 + return function() + local ret_val + local ip + repeat + i = i + 1 + ip = ip_list[i] + if ip then + local ip_dec = it.ip_to_int(ip) + if stop and (stop + 1) == ip_dec then + hosts = hosts + 1 + else + if hosts > 1 and hosts >= self.HOSTS_LIMIT then + if raw_list then + remove_items(start, stop) + end + ret_val = {[1] = start, [2] = stop} + start = ip_dec + stop = ip_dec + hosts = 1 + break + end + start = ip_dec + end + stop = ip_dec + elseif not last_call then + if hosts > 1 and hosts >= self.HOSTS_LIMIT then + if raw_list then + remove_items(start, stop) + end + ret_val = {[1] = start, [2] = stop} + last_call = true + break + end + end + until not ip + return ret_val + end +end + +function Summarize:_sort_net_list(t) + local t2 = {} + for k, v in pairs(t) do + local ip, pref = it.get_network_addr(k) + t2[#t2 + 1] = {[1] = ip, [2] = pref} + end + table.sort(t2, function(a, b) return a[1] < b[1] end) + return t2 +end + +function Summarize:_group_nets(cidr_list, raw_list) + local function remove_items(start, stop) + for i = start, stop, 256 do + local item = it.int_to_ip(i) .. "/24" + if raw_list[item] then + raw_list[item] = nil + end + end + end + + local start, stop, last_call, curr_supernet + local nets = 1 + local i = 0 + return function() + local ret_val + local cidr + repeat + i = i + 1 + cidr = cidr_list[i] + if cidr then + local network_address, prefixlen + if type(cidr) == "string" then + network_address, prefixlen = it.get_network_addr(cidr) + elseif type(cidr) == "table" then + network_address, prefixlen = cidr[1], cidr[2] + end + if prefixlen == 24 then + local supernet = it.get_supernet({[1] = network_address, [2] = prefixlen}, 16) + if stop and supernet == curr_supernet and (stop + 256) == network_address then + nets = nets + 1 + else + if nets > 1 and nets >= self.NETS_LIMIT then + if raw_list then + remove_items(start, stop) + end + ret_val = {[1] = start, [2] = stop + 255} + start = network_address + stop = network_address + nets = 1 + curr_supernet = supernet + break + end + start = network_address + curr_supernet = supernet + end + stop = network_address + end + elseif not last_call then + if nets > 1 and nets >= self.NETS_LIMIT then + if raw_list then + remove_items(start, stop) + end + ret_val = {[1] = start, [2] = stop + 255} + last_call = true + break + end + end + until not cidr + return ret_val + end +end + +function Summarize:_summarize_ranges(ip_iter) + local s_range_iter + return function() + if s_range_iter then + repeat + local ip_t = s_range_iter() + if ip_t then + return ip_t + end + until not ip_t + end + local ip_range = ip_iter() + if ip_range then + s_range_iter = it.summarize_address_range(ip_range[1], ip_range[2]) + if s_range_iter then + repeat + local ip_t = s_range_iter() + if ip_t then + return ip_t + end + until not ip_t + end + else + return + end + end +end + +function Summarize:summarize_ip_ranges(ip_list, modify_raw_list) + local summ_iter = self:_summarize_ranges( + self:_group_ip_ranges(self:_sort_ip_list(ip_list), modify_raw_list and ip_list) + ) + return function() + repeat + local ip_t = summ_iter() + if ip_t and ip_t[2] == 32 then + if modify_raw_list then + ip_list[it.int_to_ip(ip_t[1])] = true + end + else + return ip_t + end + until not ip_t + end +end + +function Summarize:summarize_nets(cidr_list, modify_raw_list) + return self:_summarize_ranges( + self:_group_nets(self:_sort_net_list(cidr_list), modify_raw_list and cidr_list) + ) +end + ------------------------------ Classes ------------------------------- local BlackListParser = Class(Config, { @@ -379,13 +572,13 @@ function BlackListParser:optimize_fqdn_table() end function BlackListParser:group_ip_ranges() - for i in si.summarize_ip_ranges(self.ip_table, true) do + for i in Summarize:summarize_ip_ranges(self.ip_table, true) do self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true end end function BlackListParser:group_cidr_ranges() - for i in si.summarize_nets(self.cidr_table, true) do + for i in Summarize:summarize_nets(self.cidr_table, true) do self.cidr_table[string.format("%s/%s", it.int_to_ip(i[1]), i[2])] = true end end diff --git a/ruantiblock-mod-py/Makefile b/ruantiblock-mod-py/Makefile index d88b624..0b0a201 100644 --- a/ruantiblock-mod-py/Makefile +++ b/ruantiblock-mod-py/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ruantiblock-mod-py PKG_VERSION:=1.3 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_MAINTAINER:=gSpot include $(INCLUDE_DIR)/package.mk @@ -32,8 +32,6 @@ endef define Package/$(PKG_NAME)/install $(INSTALL_DIR) $(1)/usr/libexec/ruantiblock $(INSTALL_BIN) ./files/usr/libexec/ruantiblock/ruab_parser.py $(1)/usr/libexec/ruantiblock/ruab_parser.py - $(INSTALL_DIR) $(1)/usr/lib/python3.11 - $(INSTALL_DATA) ./files/usr/lib/python3.11/ruab_sum_ip.py $(1)/usr/lib/python3.11/ruab_sum_ip.py $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_DATA) ./files/etc/uci-defaults/ruantiblock-mod-py $(1)/etc/uci-defaults/ruantiblock-mod-py endef diff --git a/ruantiblock-mod-py/files/usr/lib/python3.11/ruab_sum_ip.py b/ruantiblock-mod-py/files/usr/lib/python3.11/ruab_sum_ip.py deleted file mode 100644 index a0cda08..0000000 --- a/ruantiblock-mod-py/files/usr/lib/python3.11/ruab_sum_ip.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -# (с) 2020 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) - -from ipaddress import IPv4Address, IPv4Network, summarize_address_range -from typing import List, Tuple, Set, Union - -HOSTS_LIMIT: int = 0 -NETS_LIMIT: int = 0 - - -def _sort_ip_func(e: str) -> IPv4Address: - return IPv4Address(e) - - -def _group_ip_ranges(ip_list: List, raw_list: Union[List, Set] = None) -> Tuple: - def remove_items(start: IPv4Address, end: IPv4Address) -> None: - for ip in range(int(start), int(end) + 1): - raw_list.remove(str(IPv4Address(ip))) - - start = end = None - hosts = 1 - for ip in ip_list: - ip_obj = IPv4Address(ip) - if end and (end + 1) == ip_obj: - hosts += 1 - else: - if hosts > 1 and hosts >= HOSTS_LIMIT: - if raw_list: - remove_items(start, end) - yield start, end - start = ip_obj - hosts = 1 - end = ip_obj - else: - if hosts > 1 and hosts >= HOSTS_LIMIT: - if raw_list: - remove_items(start, end) - yield start, end - - -def summarize_ip_ranges(ip_list: Union[List, Set], modify_raw_list: bool = False) -> IPv4Network: - for s, e in _group_ip_ranges(sorted(ip_list, key=_sort_ip_func), - modify_raw_list and ip_list): - for i in summarize_address_range(s, e): - if i.prefixlen == 32: - if modify_raw_list: - if type(ip_list) == set: - ip_list.add(i.network_address) - else: - ip_list.append(i.network_address) - else: - yield i - - -def _sort_net_func(e: str) -> IPv4Network: - return IPv4Network(e) - - -def _group_nets(cidr_list: List, raw_list: Union[List, Set] = None) -> IPv4Network: - def remove_items(start: IPv4Address, end: IPv4Address) -> None: - for ip in range(int(start), int(end) + 1, 256): - raw_list.remove(str(IPv4Address(ip)) + "/24") - - start = end = curr_super_net = None - nets = 1 - for net in cidr_list: - net_obj = IPv4Network(net) - prefix_len = net_obj.prefixlen - if prefix_len == 24: - address = net_obj.network_address - super_net = net_obj.supernet(new_prefix=16) - if end and super_net == curr_super_net and (end + 256) == address: - nets += 1 - else: - if nets > 1 and nets >= NETS_LIMIT: - if raw_list: - remove_items(start, end) - yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) - start = address - curr_super_net = super_net - nets = 1 - end = address - else: - if nets > 1 and nets >= NETS_LIMIT: - if raw_list: - remove_items(start, end) - yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) - - -def summarize_nets(cidr_list: Union[List, Set]) -> IPv4Network: - for i in _group_nets(sorted(cidr_list, key=_sort_net_func), cidr_list): - for j in i: - yield j diff --git a/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py b/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py index 98fd95b..443c892 100755 --- a/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py +++ b/ruantiblock-mod-py/files/usr/libexec/ruantiblock/ruab_parser.py @@ -2,19 +2,19 @@ # -*- coding: utf-8 -*- """ -(с) 2020 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) +(с) 2023 gSpot (https://github.com/gSpotx2f/ruantiblock_openwrt) Python >= 3.6 """ from contextlib import contextmanager +from ipaddress import IPv4Address, IPv4Network, summarize_address_range import os import re import socket import ssl import sys from urllib import request -from ruab_sum_ip import summarize_ip_ranges, summarize_nets class Config: @@ -120,6 +120,98 @@ class Config: cls._load_filter(file_path or cls.BLLIST_IP_FILTER_FILE, cls.BLLIST_IP_FILTER_PATTERNS) +class Summarize: + HOSTS_LIMIT = 0 + NETS_LIMIT = 0 + + @staticmethod + def _sort_ip_func(e): + return IPv4Address(e) + + @classmethod + def _group_ip_ranges(cls, ip_list, raw_list=None): + + def remove_items(start, end): + for ip in range(int(start), int(end) + 1): + raw_list.remove(str(IPv4Address(ip))) + + start = end = None + hosts = 1 + for ip in ip_list: + ip_obj = IPv4Address(ip) + if end and (end + 1) == ip_obj: + hosts += 1 + else: + if hosts > 1 and hosts >= cls.HOSTS_LIMIT: + if raw_list: + remove_items(start, end) + yield start, end + start = ip_obj + hosts = 1 + end = ip_obj + else: + if hosts > 1 and hosts >= HOSTS_LIMIT: + if raw_list: + remove_items(start, end) + yield start, end + + @classmethod + def summarize_ip_ranges(cls, ip_list, modify_raw_list=False): + for s, e in cls._group_ip_ranges(sorted(ip_list, key=cls._sort_ip_func), + modify_raw_list and ip_list): + for i in summarize_address_range(s, e): + if i.prefixlen == 32: + if modify_raw_list: + if type(ip_list) == set: + ip_list.add(i.network_address) + else: + ip_list.append(i.network_address) + else: + yield i + + @staticmethod + def _sort_net_func(e): + return IPv4Network(e) + + @classmethod + def _group_nets(cls, cidr_list, raw_list=None): + + def remove_items(start, end): + for ip in range(int(start), int(end) + 1, 256): + raw_list.remove(str(IPv4Address(ip)) + "/24") + + start = end = curr_super_net = None + nets = 1 + for net in cidr_list: + net_obj = IPv4Network(net) + prefix_len = net_obj.prefixlen + if prefix_len == 24: + address = net_obj.network_address + super_net = net_obj.supernet(new_prefix=16) + if end and super_net == curr_super_net and (end + 256) == address: + nets += 1 + else: + if nets > 1 and nets >= cls.NETS_LIMIT: + if raw_list: + remove_items(start, end) + yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) + start = address + curr_super_net = super_net + nets = 1 + end = address + else: + if nets > 1 and nets >= cls.NETS_LIMIT: + if raw_list: + remove_items(start, end) + yield summarize_address_range(IPv4Address(start), IPv4Address(end + 255)) + + @classmethod + def summarize_nets(cls, cidr_list): + for i in cls._group_nets(sorted(cidr_list, key=cls._sort_net_func), cidr_list): + for j in i: + yield j + + class ParserError(Exception): def __init__(self, reason=None): super().__init__(reason) @@ -349,13 +441,13 @@ class BlackListParser(Config): def _group_ip_ranges(self): if self.BLLIST_SUMMARIZE_IP: - for i in summarize_ip_ranges(self.ip_set, True): + for i in Summarize.summarize_ip_ranges(self.ip_set, True): self.cidr_set.add(i.with_prefixlen) self.ip_count = len(self.ip_set) def _group_cidr_ranges(self): if self.BLLIST_SUMMARIZE_CIDR: - for i in summarize_nets(self.cidr_set): + for i in Summarize.summarize_nets(self.cidr_set): self.cidr_set.add(i.with_prefixlen) self.cidr_count = len(self.cidr_set)