{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# Step 1: Login to Hugging Face\n", "\n", "from huggingface_hub import login\n", "login(\"hf_XXXXJsPbaZ\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Fzuvi5bOKxFS", "trusted": true }, "outputs": [], "source": [ "# Step 2: Install required libraries\n", "!pip install accelerate\n", "!pip install python-docx\n", "!pip install PyPDF2\n", "!pip install ipywidgets\n", "!apt-get install -y antiword" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "emXo05ZVIVvq", "outputId": "f64c4eae-99ba-4fd7-b1af-f52c35ea818b", "trusted": true }, "outputs": [], "source": [ "# Step 3: Load the Gemma-3-1b-it model from Hugging Face\n", "# https://huggingface.co/google/gemma-3-1b-it\n", "\n", "import torch\n", "import torch._dynamo\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "\n", "\n", "model_id = \"google/gemma-3-1b-it\"\n", "\n", "# ✅ Use float16 for Tesla T4 to reduce memory\n", "tokenizer = AutoTokenizer.from_pretrained(model_id)\n", "model = AutoModelForCausalLM.from_pretrained(\n", " model_id,\n", " device_map=\"auto\",\n", " torch_dtype=torch.float16\n", ")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# Step 4: Configure CUDA backend and verify model device\n", "\n", "# Disable memory-efficient and flash attention (optional for debugging or compatibility)\n", "torch.backends.cuda.enable_mem_efficient_sdp(True)\n", "torch.backends.cuda.enable_flash_sdp(True)\n", "\n", "# Print which device (CPU/GPU) the model is loaded on\n", "print(f\"Model device: {model.device}\")\n", "\n", "# Confirm that setup is complete\n", "print(\"Run complete\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# ==========================\n", "# STEP 5: Define File Reading & Text Extraction Functions\n", "# ==========================\n", "\n", "import os\n", "import docx\n", "from PyPDF2 import PdfReader\n", "import subprocess\n", "\n", "# --- Function: Extract text from DOCX files ---\n", "def extract_text_from_docx(docx_path):\n", " text = \"\"\n", " doc = docx.Document(docx_path)\n", " for paragraph in doc.paragraphs:\n", " text += paragraph.text + \"\\n\"\n", " return text.strip()\n", "\n", "# --- Function: Extract text from PDF files ---\n", "def extract_text_from_pdf(pdf_path):\n", " text = \"\"\n", " with open(pdf_path, \"rb\") as file:\n", " reader = PdfReader(file)\n", " for page in reader.pages:\n", " if page.extract_text():\n", " text += page.extract_text() + \"\\n\"\n", " return text.strip()\n", "\n", "# --- Function: Extract text from DOC (97–2003) using antiword ---\n", "def extract_text_from_doc(doc_path):\n", " try:\n", " result = subprocess.run([\"antiword\", doc_path], capture_output=True, text=True)\n", " return result.stdout.strip()\n", " except Exception as e:\n", " return f\"❌ Error reading DOC file {doc_path}: {e}\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# ==========================\n", "# STEP 6 : Define response generation function\n", "# ==========================\n", "\n", "import time\n", "\n", "def generate_respone(contract_content):\n", " \"\"\"\n", " Extracts key contract information using the language model.\n", " Each category is queried separately for precise output.\n", " \"\"\"\n", " list_of_categories = [\n", " 'Contract Date', 'Effective Date', 'Renewal Term',\n", " 'Exit clause incl. notice period', 'Contract Parties',\n", " 'Documents Retention Period', 'Audit Clause', 'Audit Frequency',\n", " 'Audit Duration', 'Contract Fees', 'Payment Terms',\n", " 'Contract scope', 'KPIs', 'Service Level Agreement',\n", " 'Roles & Responsibilities', 'Deliverables', 'Contractual Reporting',\n", " 'Performance Review', 'Performance Bonus',\n", " 'Data Protection and Privacy', 'Confidentiality'\n", " ]\n", "\n", " save_dict = {}\n", " start_time = time.time()\n", "\n", " for cate in list_of_categories:\n", " prompt = (\n", " f\"Extract only the exact information about '{cate}' from the following contract. \"\n", " \"Do not explain, summarize, or rephrase. If not found, answer 'None'.\\n\\nContract:\\n\"\n", " + contract_content\n", " )\n", "\n", " chat = [{\"role\": \"user\", \"content\": prompt}]\n", " chat_template = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)\n", " inputs = tokenizer.encode(chat_template, add_special_tokens=False, return_tensors=\"pt\")\n", "\n", " outputs = model.generate(\n", " input_ids=inputs.to(model.device),\n", " do_sample=True,\n", " temperature=0.3,\n", " top_p=0.9,\n", " max_new_tokens=512\n", " )\n", "\n", " string = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", " cut_string = string.split(\"model\")[-1].strip()\n", " cut_string = cut_string.replace(\"Here’s a breakdown\", \"\").replace(\"Here's a breakdown\", \"\").strip()\n", "\n", " save_dict[cate] = cut_string\n", "\n", " print(\"✅ Extraction completed in\", round(time.time() - start_time, 2), \"seconds\")\n", " return save_dict\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# Step 6.2: Clean and organize extracted contract information\n", "\n", "import re\n", "import pandas as pd \n", "\n", "def clean_respone(respone, filename):\n", "\n", " cates = []\n", " values = []\n", " \n", " # Loop through all extracted information\n", " for key, value in respone.items():\n", " \n", " # Split text into sentences\n", " sentences = re.split(r'(?', '').replace('\\n', ' ').strip()\n", " \n", " cates.append(key)\n", " values.append(clean_text)\n", " \n", " # ✅ Create a clean, organized DataFrame\n", " data = pd.DataFrame({\n", " \"Contract name\": file_name, # <-- Auto-filled real file name\n", " \"Contract Information\": cates,\n", " \"Details\": values\n", " })\n", " \n", " # Display the result\n", " return data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# ==========================\n", "# STEP 7 : Read contract files and generate responses\n", "# ==========================\n", "\n", "# Initialize an empty list to store all response DataFrames\n", "dfs = []\n", "\n", "# Define the base directory where all contract files are stored\n", "base_dir = \"/kaggle/input/contracts\" \n", "\n", "# Iterate through each file in the dataset directory\n", "for file in os.listdir(base_dir):\n", "\n", " file_path = os.path.join(base_dir, file)\n", " file_name = os.path.basename(file).split('.')[0]\n", "\n", " print(f\"Processing file: {file}\")\n", " \n", " # --- Detect file type and extract text accordingly ---\n", " if file_path.split(\".\")[-1] == 'docx':\n", " print(\"📘 DOCX file found.\")\n", " docx_text = extract_text_from_docx(file_path)\n", " contract_content = docx_text\n", " \n", " elif file_path.split(\".\")[-1] == 'pdf':\n", " print(\"📕 PDF file found.\")\n", " pdf_text = extract_text_from_pdf(file_path)\n", " contract_content = pdf_text\n", "\n", " elif file_path.split(\".\")[-1] == \"doc\":\n", " print(\"📗 DOC (97-2003) file found.\")\n", " doc_text = extract_text_from_doc(file_path)\n", " contract_content = doc_text\n", " \n", " else:\n", " print(\"⚠️ File type not recognized. Skipping this file.\")\n", " continue\n", "\n", " # --- Step 1: Extraction completed ---\n", " print(\"✅ Text extraction complete!\")\n", "\n", " # --- Step 2: Generate model response for this contract ---\n", " print(\"🧠 Generating model response ...\")\n", " respone = generate_respone(contract_content)\n", " print(\"✅ Response generation complete!\")\n", "\n", " # --- Step 3: Clean and structure the model response ---\n", " print(\"🧹 Cleaning model response ...\")\n", " respone_data = clean_respone(respone, file_name)\n", " print(\"✅ Response cleaning complete!\")\n", " print(\"=\"*50)\n", "\n", " # Append the cleaned response to the main list\n", " dfs.append(respone_data)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# ==========================\n", "# STEP 8 : Combine, Format, and Display Final Results\n", "# ==========================\n", "\n", "import pandas as pd\n", "from IPython.display import display, HTML\n", "\n", "# 🧩 Configure pandas display options\n", "pd.set_option('display.max_rows', 50) # Limit rows displayed\n", "pd.set_option('display.max_columns', 50) # Limit columns displayed\n", "pd.set_option('display.max_colwidth', 200) # Allow wider text before truncating\n", "\n", "# 🧠 Combine all response DataFrames into a single table\n", "final_df_respone = pd.concat(dfs, ignore_index=True)\n", "\n", "print(\"✅ All responses combined successfully!\")\n", "print(f\"Total contracts processed: {len(dfs)}\")\n", "print(\"------------------------------------------------------------\")\n", "\n", "# 🎨 Create custom CSS for better table visualization\n", "custom_css = \"\"\"\n", "\n", "\"\"\"\n", "\n", "# 👀 Display scrollable HTML table with clean formatting\n", "print(\"📋 Formatted full table preview (scroll horizontally if needed):\")\n", "html_table = final_df_respone.to_html(index=False, escape=False)\n", "display(HTML(custom_css + f\"\"\"\n", "