Extra/detect_lexemes_token_from_input.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "input_statement = \"position = initial + rate * 60\"\n",
    "print(input_statement)\n",
    "\n",
    "lexemes = re.findall(r'\\b\\w+\\b|[=+\\-*\\\\/();]', input_statement)\n",
    "\n",
    "print(\"Lexemes:\", lexemes)\n",
    "\n",
    "print()\n",
    "\n",
    "\n",
    "\n",
    "def token_identifier(expression):\n",
    "    keywords = [\"int\",\"if\",\"else\",\"while\",\"do\",\"break\",\"continue\",\"double\",\"float\",\"return\",\"char\",\"case\",\"sizeof\",\"long\",\"short\",\"typeof\",\"switch\",\"unsigned\",\"void\",\"static\",\"struct\",\"goto\"]\n",
    "    identifiers = []\n",
    "    constants = []\n",
    "    symbols = [\"=\", \"+\", \"-\", \"*\", \"/\", \"(\", \")\", \";\"]\n",
    "\n",
    "    tokens = re.findall(\"\\w+|\\d+|[=+\\-*\\\\/();]\", expression)\n",
    "    for token in tokens:\n",
    "        if token in keywords:\n",
    "            print(f\"{token} is a keyword\")\n",
    "        elif token in symbols:\n",
    "            print(f\"{token} is a symbol\")\n",
    "        elif token.isdigit():\n",
    "            print(f\"{token} is a constant\")\n",
    "        else:\n",
    "            if token[0].isdigit():\n",
    "                print(f\"{token} is not a valid identifier\")\n",
    "            else:\n",
    "                print(f\"{token} is an identifier\")\n",
    "\n",
    "input_string = \"int a = b + 10 c + 7\"\n",
    "print(input_string)\n",
    "token_identifier(input_string)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def tokenize(input_string):\n",
    "    # Define regular expression patterns\n",
    "    keyword_pattern = re.compile(r'\\b(int|float|char)\\b')\n",
    "    identifier_pattern = re.compile(r'\\b[a-zA-Z_]\\w*\\b')\n",
    "    constant_pattern = re.compile(r'\\b\\d+\\b')\n",
    "    string_literal_pattern = re.compile(r'\\\".*?\\\"')\n",
    "    symbol_pattern = re.compile(r'[-+*/=(),;]')\n",
    "\n",
    "    # Tokenize the input string\n",
    "    tokens = []\n",
    "    current_position = 0\n",
    "\n",
    "    while current_position < len(input_string):\n",
    "        # Skip whitespace\n",
    "        while current_position < len(input_string) and input_string[current_position].isspace():\n",
    "            current_position += 1\n",
    "\n",
    "        # Check for each pattern\n",
    "        match = None\n",
    "        for pattern, token_type in [(keyword_pattern, 'Keyword'),\n",
    "                                    (identifier_pattern, 'Identifier'),\n",
    "                                    (constant_pattern, 'Constant'),\n",
    "                                    (string_literal_pattern, 'String Literal'),\n",
    "                                    (symbol_pattern, 'Symbol')]:\n",
    "            match = pattern.match(input_string, current_position)\n",
    "            if match:\n",
    "                tokens.append((match.group(), token_type))\n",
    "                current_position = match.end()\n",
    "                break\n",
    "\n",
    "        if not match:\n",
    "            print(f\"Error: Unexpected character at position {current_position}\")\n",
    "            break\n",
    "\n",
    "    return tokens\n",
    "\n",
    "# Example usage\n",
    "input_statement = \"int a = b + 10c + 7\"\n",
    "tokens = tokenize(input_statement)\n",
    "\n",
    "# Display the detected tokens\n",
    "for token, token_type in tokens:\n",
    "    print(f\"Token: {token}, Type: {token_type}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}