Untitled

# -*- coding: utf-8 -*-

import os
import ipaddress

input_file = "Input.txt"
output_file = "good2.txt"
honeypoints_file = "honeypoints2.txt"
chunk_size = 1000000

def process_block(block, unique_ips, output_file, honeypoints_file):
    new_lines = []
    duplicate_ips = set()

    for line in block:
        ip_port = line.strip()
        ip = ip_port.split(":")[0]  # Получаем только IP
        if ip not in unique_ips and ip not in duplicate_ips:
            new_lines.append(ip_port)
            unique_ips.add(ip)
        else:
            duplicate_ips.add(ip)

    with open(output_file, 'a', encoding='utf-8') as file:
        file.write('\n'.join(new_lines) + '\n')

    with open(honeypoints_file, 'a', encoding='utf-8') as honeypoints:
        duplicate_masks = [str(ipaddress.IPv4Network(f'{ip}/24', strict=False)) for ip in duplicate_ips]
        honeypoints.write('\n'.join(duplicate_masks) + '\n')

def process_file(input_file, output_file, honeypoints_file, chunk_size):
    unique_ips = set()

    with open(input_file, 'r', encoding='utf-8') as file:
        block = []
        for line in file:
            block.append(line)
            if len(block) >= chunk_size:
                process_block(block, unique_ips, output_file, honeypoints_file)
                block = []

        if block:
            process_block(block, unique_ips, output_file, honeypoints_file)

if __name__ == "__main__":
    process_file(input_file, output_file, honeypoints_file, chunk_size)