niracler

import csv
import socks
from telethon import TelegramClient
from telethon.tl.functions.messages import GetHistoryRequest

# Set up TelegramClient and connect to the Telegram API
client = TelegramClient(
    'demo',
    'api_id',
    'api_hash',
    proxy=(socks.SOCKS5, '127.0.0.1', 1080)
)

async def export_to_csv(filename, fieldnames, data):
    """
    Export data to a CSV file.

    Parameters:
    filename -- Name of the export file
    fieldnames -- List of CSV header field names
    data -- List of dictionaries to export
    """
    with open(filename, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(data)

async def fetch_messages(channel_username):
    """
    Fetch all messages from the specified channel.

    Parameters:
    channel_username -- Username of the target channel
    """
    channel_entity = await client.get_input_entity(channel_username)
    offset_id = 0  # Initial message ID offset
    all_messages = []  # List to store all messages

    while True:
        # Request message history
        history = await client(GetHistoryRequest(
            peer=channel_entity,
            offset_id=offset_id,
            offset_date=None,
            add_offset=0,
            limit=100,  # Number of messages to request at a time
            max_id=0,
            min_id=0,
            hash=0
        ))
        if not history.messages:  # End loop when there are no more messages
            break

        for message in history.messages:
            if message.message:  # Only process messages with text content
                # Serialize message to dictionary form
                message_dict = {
                    'id': message.id,
                    'date': message.date.strftime('%Y-%m-%d %H:%M:%S'),
                    'text': message.message
                }
                all_messages.append(message_dict)
        offset_id = history.messages[-1].id
        print(f"Fetched messages: {len(all_messages)}")
    return all_messages

async def main():
    """
    Main program: Fetch messages from the specified channel and save to a CSV file.
    """
    await client.start()  # Start the Telegram client
    print("Client Created")

    channel_username = 'niracler_channel'  # Username of the Telegram channel you want to scrape
    all_messages = await fetch_messages(channel_username)  # Fetch messages

    # Define CSV file headers and export
    headers = ['id', 'date', 'text']
    await export_to_csv('channel_messages.csv', headers, all_messages)

# When this script is run as the main program
if __name__ == '__main__':
    client.loop.run_until_complete(main())

python telegram_to_csv.py

import pandas as pd
from openai import OpenAI

# Configure OpenAI client
client = OpenAI(api_key='YOUR_API_KEY')

def get_embedding(text, model="text-embedding-ada-002"):
    """
    Get the embedding vector for the text.
    """
    text = text.replace("\n", " ")  # Clean newline characters from text
    response = client.embeddings.create(input=[text], model=model)  # Request embedding vector
    return response.data[0].embedding  # Extract and return the embedding vector

def embedding_gen():
    """
    Generate embedding vector data for tutorial text.
    """
    df = pd.read_csv('channel_messages.csv')  # Read CSV file into DataFrame
    df['text_with_date'] = df['date'] + " " + df['text']  # Concatenate date and text
    df['ada_embedding'] = df[:100].text_with_date.apply(get_embedding)  # Apply text embedding function in batches

    del df['text_with_date']  # Delete 'text_with_date' column
    df.to_csv('embedded_1k_reviews.csv', index=False)  # Save results to a new CSV file
    
    # Print the first few rows of the DataFrame for confirmation
    print(df.head())

# When the script is run directly
if __name__ == "__main__":
    embedding_gen()

python embedding_generator.py

import ast
import sys
import pandas as pd
import numpy as np
from tabulate import tabulate
from openai import OpenAI

# Configure OpenAI client
client = OpenAI(api_key='YOUR_API_KEY')

def get_embedding(text, model="text-embedding-ada-002"):
    """
    Get the embedding vector for the text.
    """
    text = text.replace("\n", " ")  # Clean newline characters from text
    response = client.embeddings.create(input=[text], model=model)  # Request embedding vector
    return response.data[0].embedding  # Extract and return the embedding vector

def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

def embedding_search(query, df, model="text-embedding-ada-002"):
    """
    Use OpenAI API to search for embedding vectors.
    """
    query_embedding = get_embedding(query, model=model)  # Get embedding vector for the query text
    df['similarity'] = df.ada_embedding.apply(lambda x: cosine_similarity(ast.literal_eval(x), query_embedding))  # Calculate similarity
    df = df.sort_values(by='similarity', ascending=False)  # Sort by similarity in descending order
    df = df.drop(columns=['ada_embedding'])  # Remove embedding vector column
    return df

if __name__ == "__main__":
    df = pd.read_csv('embedded_1k_reviews.csv')  # Read CSV file into DataFrame
    query = sys.argv[1]
    df = embedding_search(query, df)  # Search for embedding vectors
    print(tabulate(df.head(10), headers='keys', tablefmt='psql'))  # Print the top 10 results

$ python embedding_search.py Animal Crossing
+------+---------------------+--------------------------------------------------------------+----------------+
|      | date                | text                                                         |   similarities |
|------+---------------------+--------------------------------------------------------------+----------------|
| 1041 | 2021-04-03 06:18:40 | Neil's Animal Crossing                                       |       0.843896 |
|  836 | 2021-10-16 02:37:16 | Animal Crossing Direct Chinese Video                         |       0.826405 |
|      |                     | https://www.youtube.com/watch?v=rI_jWfNd2dc                  |                |
| 1208 | 2019-11-10 00:05:56 | Raising animals seems very interesting                       |       0.822377 |
|  489 | 2023-06-16 09:33:15 | Watching the life of a kitten reminds me of Sisyphus in mythology |       0.802677 |
|  369 | 2023-08-16 02:15:54 | Do house cats get bored and lonely?                          |       0.797062 |
|   13 | 2023-12-14 13:17:59 | Attended 🤗                                                  |       0.796492 |
| 1177 | 2020-02-12 10:27:45 | The reason why people eat wild animals repeatedly is related to the deeply rooted concepts in traditional Chinese medicine |       0.796363 |
|      |                     | Health preservation, dietary therapy, supplementation, medicinal cuisine, shape complementing shape, nourishing qi and blood... |                |
|      |                     | Pseudoscience is reviving, and if not curbed now, similar things will happen in the future. |                |
|      |                     | Science is the only way.                                     |                |
|  801 | 2021-11-07 13:46:21 | I didn't expect that this year's game of the year would still be Animal Crossing and Fire Emblem. |       0.796246 |
|      |                     | Animal Crossing is because I didn't play enough before, Fire Emblem is because of a major event that made me want to replay it. |                |
|  837 | 2021-10-16 02:37:16 | No way, is my game of the year going to be Animal Crossing again? |       0.795871 |
|  423 | 2023-07-29 14:11:22 | A profile picture that can be called spiritual pollution~~   |       0.794144 |
+------+---------------------+--------------------------------------------------------------+----------------+