Masud_Sir/recognize_token_from_lexemes_with_regex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "- `'IDENTIFIER'`: This pattern uses `^[a-zA-Z_][a-zA-Z0-9_]*$`. \n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `[a-zA-Z_]`: Matches a single letter or underscore at the beginning.\n",
    "  - `[a-zA-Z0-9_]*`: Matches zero or more occurrences of letters, digits, or underscores.\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "- `'INTEGER'`: This pattern uses `^[0-9]+$`.\n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `[0-9]+`: Matches one or more digits.\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "- `'FLOAT'`: This pattern uses `^[0-9]+\\.[0-9]+$`.\n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `[0-9]+\\.[0-9]+`: Matches digits, a dot, and more digits (representing a simple float).\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "- `'OPERATOR'`: This pattern uses `^[+\\-*/%]$`.\n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `[+\\-*/%]`: Matches a single character among +, -, *, /, or %.\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "- `'PUNCTUATION'`: This pattern uses `^[;,()]$`.\n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `[;,()]`: Matches a single character among ;, ,, (, or ).\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "- `'KEYWORD'`: This pattern uses `^(if|else|while|for|int|float|return|def|class|import|from)$`.\n",
    "  - `^`: Asserts the start of the string.\n",
    "  - `(if|else|while|for|int|float|return|def|class|import|from)`: Matches one of the specified keywords.\n",
    "  - `$`: Asserts the end of the string.\n",
    "\n",
    "These regular expressions help define patterns for different types of tokens, allowing the program to recognize and categorize lexemes based on these patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def recognize_tokens(lexeme):\n",
    "    # Define regular expressions for different types of tokens\n",
    "    token_patterns = {\n",
    "    'IDENTIFIER': r'^[a-zA-Z_][a-zA-Z0-9_]*$',  # Matches valid identifiers (start with a letter or underscore, \n",
    "                                                # followed by letters, digits, or underscores)\n",
    "    'INTEGER': r'^[0-9]+$',  # Matches integers (one or more digits)\n",
    "    'FLOAT': r'^[0-9]+\\.[0-9]+$',  # Matches floats (digits followed by a dot and more digits)\n",
    "    'OPERATOR': r'^[+\\-*/%]$',  # Matches operators (+, -, *, /, %)\n",
    "    'PUNCTUATION': r'^[;,()]$',  # Matches common punctuation symbols (;, ,, (, ))\n",
    "    'KEYWORD': r'^(if|else|while|for|int|float|return|def|class|import|from)$'  # Matches specific keywords\n",
    "    # Add more token patterns as needed\n",
    "}\n",
    "\n",
    "\n",
    "    # Check each token pattern and return the corresponding token type if matched\n",
    "    for token_type, pattern in token_patterns.items():\n",
    "        if re.match(pattern, lexeme):\n",
    "            return token_type\n",
    "\n",
    "    # If no pattern is matched, return 'UNKNOWN'\n",
    "    return 'UNKNOWN'\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    user_input = input(\"Enter one lexeme: \")\n",
    "\n",
    "    # Recognize the token type for the given lexeme\n",
    "    token_type = recognize_tokens(user_input)\n",
    "\n",
    "    # Display the result\n",
    "    print(f\"The lexeme '{user_input}' represents a '{token_type}' token.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}