Untitled

class FilterImportantLogsDoFn(beam.DoFn):
    def __init__(self, important_words_file):
        self.important_words_file = important_words_file

    def process(self, df):
        with open(self.important_words_file, "r") as f:
            important_words = f.read().splitlines()
        filtered_df = df[df['log_text'].str.contains('|'.join(important_words))]
        yield filtered_df