From 855a3eb95119176cef19c5246cf6181a6fdef7ed Mon Sep 17 00:00:00 2001 From: mmalcek Date: Thu, 7 Mar 2024 10:38:07 +0100 Subject: [PATCH] chatGPT integration --- README.md | 21 +++-- docs/README.md | 198 +++++++++++++++++++++++++++++++---------------- docs/examples.md | 107 +++++++++++++++++++++---- go.mod | 2 + go.sum | 4 +- main.go | 65 +++++++++++++++- main_test.go | 7 ++ 7 files changed, 312 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 8dcf020..4338962 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CodeQL](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/mmalcek/bafi)](https://goreportcard.com/report/github.com/mmalcek/bafi) [![License](https://img.shields.io/github/license/mmalcek/bafi)](https://github.com/mmalcek/bafi/blob/main/LICENSE) -[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go#text-processing) +[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go#text-processing) [![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/mmalcek/bafi?label=latest%20release)](https://github.com/mmalcek/bafi/releases/latest) # Universal JSON, BSON, YAML, CSV, XML, mt940 translator to ANY format using templates @@ -10,31 +10,40 @@ ## Key features + - Various input formats **(json, bson, yaml, csv, xml, mt940)** - Flexible output formatting using text templates - Support for [Lua](https://www.lua.org/pil/contents.html) custom functions which allows very flexible data manipulation - stdin/stdout support which allows get data from source -> translate -> delivery to destination. This allows easily translate data between different web services like **REST to SOAP, SOAP to REST, REST to CSV, ...** - Merge multiple input files in various formats into single output file formated using template - - +- Support chatGPT queries to analyze or format data (experimental) ## Documentation [https://mmalcek.github.io/bafi/](https://mmalcek.github.io/bafi/) ## Releases (Windows, MAC, Linux) [https://github.com/mmalcek/bafi/releases](https://github.com/mmalcek/bafi/releases) -usage: +usage: + ``` bafi.exe -i testdata.xml -t template.tmpl -o output.txt ``` -or + +or + ``` curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}" ``` +or + +``` +curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "What's the current CZK rate?" +``` + More examples and description in [documentation](https://mmalcek.github.io/bafi/) **If you like this app you can buy me a coffe ;)** Buy Me a Coffee at ko-fi.com - \ No newline at end of file + diff --git a/docs/README.md b/docs/README.md index 66157ad..1e7799d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,19 +1,24 @@ # BaFi + **Universal JSON, BSON, YAML, CSV, XML, mt940 translator to ANY format using templates** **Github repository** + - [https://github.com/mmalcek/bafi](https://github.com/mmalcek/bafi) **Releases (Windows, MAC, Linux)** + - [https://github.com/mmalcek/bafi/releases](https://github.com/mmalcek/bafi/releases) ## Key features + - Various input formats **(json, bson, yaml, csv, xml, mt940)** - Flexible output formatting using text templates - Output can be anything: HTML page, SQL Query, Shell script, CSV file, ... - Support for [Lua](https://www.lua.org/pil/contents.html) custom functions which allows very flexible data manipulation - stdin/stdout support which allows get data from source -> translate -> delivery to destination. This allows easily translate data between different web services like **REST to SOAP, SOAP to REST, REST to CSV, ...** - Merge multiple input files in various formats into single output file formated using template +- Support chatGPT queries to analyze or format data (experimental) @@ -21,8 +26,9 @@ [![CodeQL](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/mmalcek/bafi)](https://goreportcard.com/report/github.com/mmalcek/bafi) [![License](https://img.shields.io/github/license/mmalcek/bafi)](https://github.com/mmalcek/bafi/blob/main/LICENSE) -[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go#text-processing) +[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go#text-processing) [![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/mmalcek/bafi?label=latest%20release)](https://github.com/mmalcek/bafi/releases/latest) + **If you like this app you can buy me a coffe ;)** @@ -32,41 +38,52 @@ ## How does it work? + Application automaticaly parse input data into object which can be simply accessed in tamplate using dot notation where first dot represent root of object **{{ . }}**. For example JSON document **myUser.json** + ```json { - "user": { - "name": "John Doe", - "age": 25, - "address": { - "street": "Main Street", - "city": "New York", - "state": "NY" - }, - "favourite_colors": ["red", "green", "blue"] - } + "user": { + "name": "John Doe", + "age": 25, + "address": { + "street": "Main Street", + "city": "New York", + "state": "NY" + }, + "favourite_colors": ["red", "green", "blue"] + } } ``` + - Get user name: + ```sh bafi.exe -i myUser.json -t '?{{.user.name}}' ``` + - Use function to change all letters to uppercase: + ```sh bafi.exe -i myUser.json -t '?{{upper .user.name}}' ``` + - Use IF statement to compare user age to 20: + ```sh bafi.exe -i myUser.json -t '?User is {{if gt (toInt .user.age) 20}}old{{else}}young{{end}}.' ``` - List favourite colors: + ```sh bafi.exe -i myUser.json -t '?{{range .user.favourite_colors}}{{.}},{{end}}' ``` + - Format data using template file **myTemplate.tmpl** and save output to **myUser.txt**: + ```sh bafi.exe -i myUser.json -t myTemplate.tmpl -o myUser.txt ``` @@ -83,6 +100,7 @@ Favourite colors: {{$colors}} ``` note: in Powershell you must use .\\bafi.exe e.g. + ```powershell .\bafi.exe -i input.csv -t "?{{toXML .}}" curl.exe -s someurl.com/api/xxx | .\bafi.exe -f json -t "?{{toXML .}}" @@ -106,164 +124,208 @@ function getBAFI() { ## Online demo (WASM) - **Just try it here :)** + +**Just try it here :)**
+

 
 ## Command line arguments
-- **-i input.xml** Input file name. 
-    - If not defined app tries read stdin
-    - If prefixed with "?" (**-i ?files.yaml**) app will expect yaml file with multiple files description. See [example](examples/#multiple-input-files)
-- **-o output.txt** Output file name. 
-    - If not defined result is send to stdout
-- **-t template.tmpl** Template file. Alternatively you can use *inline* template 
-    - inline template must start with **?** e.g. -t **"?{{.someValue}}"**
-- **-f json** Input format. 
-    - Supported formats: **json, bson, yaml, csv, xml, mt940**
-    - If not defined (for file input) app tries detect input format automatically by file extension
-- **-d ','** Data delimiter 
-    - format CSV:
-        - Can be defined as string e.g. -d ',' or as [hex](https://www.asciitable.com/asciifull.gif) value prefixed by **0x** e.g. 'TAB' can be defined as -f 0x09. Default delimiter is comma (**,**) 
-    - format mt940:
-        - For Multiple messages in one file (e.g. Multicash). Can be defined as string e.g. -d "-\}\r\n" or "\r\n$" . If delimiter is set BaFi will return array of mt940 messages
+
+- **-i input.xml** Input file name.
+  - If not defined app tries read stdin
+  - If prefixed with "?" (**-i ?files.yaml**) app will expect yaml file with multiple files description. See [example](examples/#multiple-input-files)
+- **-o output.txt** Output file name.
+  - If not defined result is send to stdout
+- **-t template.tmpl** Template file. Alternatively you can use _inline_ template
+  - inline template must start with **?** e.g. -t **"?{{.someValue}}"**
+- **-f json** Input format.
+  - Supported formats: **json, bson, yaml, csv, xml, mt940**
+  - If not defined (for file input) app tries detect input format automatically by file extension
+- **-d ','** Data delimiter
+  - format CSV:
+    - Can be defined as string e.g. -d ',' or as [hex](https://www.asciitable.com/asciifull.gif) value prefixed by **0x** e.g. 'TAB' can be defined as -f 0x09. Default delimiter is comma (**,**)
+  - format mt940:
+    - For Multiple messages in one file (e.g. Multicash). Can be defined as string e.g. -d "-\}\r\n" or "\r\n$" . If delimiter is set BaFi will return array of mt940 messages
 - **-v** Show current verion
 - **-?** list available command line arguments
+- **-gk myChatGPTToken** - ChatGPT token
+- **-gq "What's the current CZK rate?"** - ChatGPT query
+- **-gm gpt35** - ChatGPT model. Currently supportsed options "gpt35"(default), "gpt4"
 
 ```sh
 bafi.exe -i testdata.xml -t template.tmpl -o output.txt
 ```
+
 More examples [here](examples/#command-line)
+
 ## Templates
-Bafi uses [text/template](https://pkg.go.dev/text/template). Here is a quick summary how to use. Examples are based on *testdata.xml* included in project
+
+Bafi uses [text/template](https://pkg.go.dev/text/template). Here is a quick summary how to use. Examples are based on _testdata.xml_ included in project
 
 note: in **vscode** you can use [gotemplate-syntax](https://marketplace.visualstudio.com/items?itemName=casualjim.gotemplate) for syntax highlighting
 
 ### Comments
+
 ```
 {{/* a comment */}}
 {{- /* a comment with white space trimmed from preceding and following text */ -}}
 ```
 
 ### Trim new line
+
 New line before or after text can be trimmed by adding dash
+
 ```
 {{- .TOP_LEVEL}}, {{.TOP_LEVEL -}}
 ```
+
 ### Accessing data
-Data are accessible by *pipline* which is represented by dot
 
-- Simplest template 
+Data are accessible by _pipline_ which is represented by dot
+
+- Simplest template
+
 ```
 {{.}}
 ```
-- Get data form inner node 
+
+- Get data form inner node
+
 ```
 {{.TOP_LEVEL}}
 ```
+
 - Get data from XML tag. XML tags are autoprefixed by dash and accessible as index
+
 ```
 {{index .TOP_LEVEL "-description"}}
 ```
+
 - Convert TOP_LEVEL node to JSON
+
 ```
 {{toJSON .TOP_LEVEL}}
 ```
 
 ### Variables
+
 You can store selected data to [template variable](https://pkg.go.dev/text/template#hdr-Variables)
+
 ```
 {{$myVar := .TOP_LEVEL}}
 ```
+
 ### Actions
+
 Template allows to use [actions](https://pkg.go.dev/text/template#hdr-Actions), for example
 
 Iterate over lines
+
 ```
 {{range .TOP_LEVEL.DATA_LINE}}{{.val1}}{{end}}
 ```
-If statement 
+
+If statement
+
 ```
 {{if gt (int $val1) (int $val2)}}Value1{{else}}Value2{{end}} is greater
 ```
 
 ### Functions
+
 In go templates all operations are done by functions where function name is followed by operands
 
-For example: 
+For example:
 
 count val1+val2
+
 ```
 {{add $val1 $val2}}
 ```
+
 count (val1+val2)/val3
+
 ```
 {{div (add $val1 $val2) $val3}}
 ```
 
-This is called [Polish notation](https://en.wikipedia.org/wiki/Polish_notation) or "Prefix notation" also used in another languages like [Lisp](https://en.wikipedia.org/wiki/Lisp_(programming_language)) 
+This is called [Polish notation](https://en.wikipedia.org/wiki/Polish_notation) or "Prefix notation" also used in another languages like [Lisp]()
 
-The key benefit of using this notation is that order of operations is clear. For example **6/2*(1+2)** - even diferent calculators may have different opinion on order of operations in this case. With Polish notation order of operations is strictly defined (from inside to outside) **div 6 (mul 2 (add 1 2))** . This brings benefits with increasing number of operations especially in templates where math and non-math operations can be mixed together.
+The key benefit of using this notation is that order of operations is clear. For example **6/2\*(1+2)** - even diferent calculators may have different opinion on order of operations in this case. With Polish notation order of operations is strictly defined (from inside to outside) **div 6 (mul 2 (add 1 2))** . This brings benefits with increasing number of operations especially in templates where math and non-math operations can be mixed together.
 
-  
+ 
 
 For example we have json array of items numbered from **0**
 
 ```json
-{"items": ["item-0","item-1","item-2","item-3"]}
+{ "items": ["item-0", "item-1", "item-2", "item-3"] }
 ```
+
 We need change items numbering to start with **1**. To achieve this we have to do series of operations: 1. trim prefix "item-" -> 2. convert to int -> 3. add 1 -> 4. convert to string -> 5. append "item-" for all items in range. This can be done in one line
+
 ```
 {{ range .items }}{{ print "item-" (toString (add1 (toInt (trimPrefix . "item-")))) }} {{ end }}
 ```
+
 or alternatively (slightly shorter) print formatted string - examples [here](https://zetcode.com/golang/string-format/), documentation [here](https://golang.org/pkg/fmt/)
+
 ```
 {{ range .items }}{{ printf "item-%d " (add1 (toInt (trimPrefix . "item-"))) }}{{ end }}
 ```
+
 but BaFi also tries automaticaly cast variables so the shortest option is
+
 ```
 {{range .items}}{{print "item-" (add1 (trimPrefix . "item-"))}} {{end}}
 ```
-Expected result: **item-1 item-2 item-3 item-4** 
+
+Expected result: **item-1 item-2 item-3 item-4**
 
 There are 3 categories of functions
 
 #### Native functions
+
 text/template integrates [native functions](https://pkg.go.dev/text/template#hdr-Functions) to work with data
 
 #### Additional functions
+
 Asside of integated functions bafi contains additional common functions
+
 ##### Math functions
+
 - **add** - {{add .Value1 .Value2}}
 - **add1** - {{add1 .Value1}} = Value1+1
 - **sub** - substract
@@ -285,15 +347,16 @@ Asside of integated functions bafi contains additional common functions
 ##### Date functions
 
 - **dateFormat** - {{dateFormat .Value "oldFormat" "newFormat"}} - [GO time format](https://programming.guide/go/format-parse-string-time-date-example.html)
-    - {{dateFormat "2021-08-26T22:14:00" "2006-01-02T15:04:05" "02.01.2006-15:04"}}
-- **dateFormatTZ** - {{dateFormatTZ .Value "oldFormat" "newFormat" "timeZone"}} 
-    - This fuction is similar to dateFormat but applies timezone offset - [Timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
-    - {{dateFormatTZ "2021-08-26T03:35:00.000+04:00" "2006-01-02T15:04:05.000-07:00" "02.01.2006-15:04" "Europe/Prague"}}
+  - {{dateFormat "2021-08-26T22:14:00" "2006-01-02T15:04:05" "02.01.2006-15:04"}}
+- **dateFormatTZ** - {{dateFormatTZ .Value "oldFormat" "newFormat" "timeZone"}}
+  - This fuction is similar to dateFormat but applies timezone offset - [Timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
+  - {{dateFormatTZ "2021-08-26T03:35:00.000+04:00" "2006-01-02T15:04:05.000-07:00" "02.01.2006-15:04" "Europe/Prague"}}
 - **dateToInt** - {{dateToInt .Value "dateFormat"}} - convert date to integer (unixtime, int64), usefull for comparing dates
 - **intToDate** - {{intToDate .Value "dateFormat"}} - convert integer (unixtime, int64) to date, usefull for comparing dates
 - **now** - {{now "02.01.2006"}} - GO format date (see notes below)
 
 ##### String functions
+
 - **addSubstring** - {{addSubstring $myString, "XX", $position}} add substring to $position in string (if $position is 1,2,3 = Adding from right, if -1,-2,-3 = Adding from left)
 - **atoi** - {{atoi "042"}} - string to int. Result will be 42. atoi must be used especially for convert strings with leading zeroes
 - **b64enc** - encode to base64
@@ -317,12 +380,12 @@ Asside of integated functions bafi contains additional common functions
 - **trimSuffix** - {{trimSuffix "!Hello World!" "!"}} - returns "!HelloWorld"
 - **mapJSON** - convert stringified JSON to map so it can be used as object or translated to other formats (e.g. "toXML"). Check template.tmpl for example
 - **mustArray** - {{mustArray .Value1}} - convert to array. Useful with XML where single node is not treated as array
-- **toBool** - {{toBool "true"}} - string to bool 
+- **toBool** - {{toBool "true"}} - string to bool
 - **toDecimal** - {{toDecimal "3.14159"}} - cast to decimal (if error return 0)
 - **toDecimalString** - {{toDecimalString "3.14159"}} - cast to decimal string (if error return "error message")
 - **toFloat64** - {{float64 "3.14159"}} - cast to float64
-- **toInt** - {{int true}} - cast to int. Result will be 1. If you need convert string with leading zeroes use "atoi" 
-- **toInt64** - {{int64 "42"}} - cast to int64. Result will be 42. If you need convert string with leading zeroes use "atoi" 
+- **toInt** - {{int true}} - cast to int. Result will be 1. If you need convert string with leading zeroes use "atoi"
+- **toInt64** - {{int64 "42"}} - cast to int64. Result will be 42. If you need convert string with leading zeroes use "atoi"
 - **toString** - {{toString 42}} - int to string
 - **toJSON** - convert input object to JSON
 - **toBSON** - convert input object to BSON
@@ -332,11 +395,12 @@ Asside of integated functions bafi contains additional common functions
 - **upper** - to uppercase
 - **uuid** - generate UUID
 
-
 #### Lua custom functions
+
 You can write your own custom lua functions defined in ./lua/functions.lua file
 
 Call Lua function in template ("sum" - Lua function name)
+
 ```
 {{lua "sum" .val1 .val2}}
 ```
@@ -347,14 +411,14 @@ Call Lua function in template ("sum" - Lua function name)
 - Lua [documentation](http://www.lua.org/manual/5.1/)
 
 Minimal functions.lua example
+
 ```lua
 json = require './lua/json'
 
-function sum(incomingData) 
+function sum(incomingData)
     dataTable = json.decode(incomingData)
     return tostring(tonumber(dataTable[1]) + tonumber(dataTable[2]))
 end
 ```
 
 Check [examples](examples/) and **template.tmpl** and **testdata.xml** for advanced examples
-
diff --git a/docs/examples.md b/docs/examples.md
index 5323853..828364c 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -1,45 +1,65 @@
 # Examples
+
 ## Command line
+
 note: in Powershell you must use .\\bafi.exe e.g.
+
 ```powershell
 .\bafi.exe -i input.csv -t "?{{toXML .}}"
 curl.exe -s someurl.com/api/xxx | .\bafi.exe -f json -t "?{{toXML .}}"
 ```
-### Basic  
-Get data from *testdata.xml* -> process using *template.tmpl* -> save output as *output.txt*
+
+### Basic
+
+Get data from _testdata.xml_ -> process using _template.tmpl_ -> save output as _output.txt_
+
 ```sh
 bafi.exe -i testdata.xml -t template.tmpl -o output.txt
 ```
+
 ### Inline template
-Get data from *testdata.xml* -> process using inline template -> save output as *output.json*
+
+Get data from _testdata.xml_ -> process using inline template -> save output as _output.json_
+
 ```sh
 bafi.exe -i testdata.xml -o output.json -t "?{{toJSON .}}"
 ```
+
 note: BaFi inline template must start with **?** e.g. **"?{{toJSON .}}"**
 [How to format inline templates](https://pkg.go.dev/text/template#hdr-Examples)
 
 ### Stdin/REST
-Get data from REST api -> convert to XML -> output to Stdout. 
+
+Get data from REST api -> convert to XML -> output to Stdout.
+
 ```sh
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}"
 ```
+
 More info about curl [here](https://curl.se/) but you can of course use any tool with stdout
 
 ### Append output file
+
 Redirect stdout to file and append ( > = replace, >> = apppend )
+
 ```sh
 bafi.exe -i testdata.xml -t template.tmpl >> output.txt
 ```
 
 ## Template
+
 Examples are based on testdata.tmpl included in project
 
 ### XML to CSV
+
 - command
+
 ```sh
 bafi.exe -i testdata.xml -t myTemplate.tmpl -o output.csv
 ```
+
 - myTemplate.tmpl
+
 ```
 Employee,Date,val1,val2,val3,SUM,LuaMultiply,linkedText
 {{- range .TOP_LEVEL.DATA_LINE}}
@@ -52,11 +72,15 @@ Employee,Date,val1,val2,val3,SUM,LuaMultiply,linkedText
 ```
 
 ### JSON to CSV
+
 - command
+
 ```sh
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.html
 ```
+
 - myTemplate.tmpl
+
 ```
 name,surname
 {{- range .customers}}
@@ -65,11 +89,15 @@ name,surname
 ```
 
 ### JSON to HTML
+
 - command
+
 ```sh
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.html
 ```
+
 - myTemplate.tmpl
+
 ```
 
     
@@ -86,12 +114,17 @@ curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTempl
 table, th, td { border: 1px solid black; width: 400px; }
 
 ```
+
 ### JSON to custom XML
-- command 
+
+- command
+
 ```sh
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.xml
 ```
+
 - myTemplate.tmpl
+
 ```
 
 
@@ -105,12 +138,15 @@ curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTempl
 ```
 
 ### XML to custom JSON
-- command 
+
+- command
+
 ```sh
 bafi.exe -i testdata.xml -t myTemplate.tmpl -o output.json
 ```
 
 - myTemplate.tmpl
+
 ```
 {{- $new := "{\"employees\": [" }}
 {{- range .TOP_LEVEL.DATA_LINE}}
@@ -120,57 +156,73 @@ bafi.exe -i testdata.xml -t myTemplate.tmpl -o output.json
 {{- $new = print (trimSuffix $new "," ) "]}"}}
 {{ $new}}
 ```
+
 JSON in $new variable can be mapped to struct and autoformatted to other formats like:
 
 - Transform $new to YAML
+
 ```
 {{toYAML (mapJSON $new) -}}
 ```
+
 - Transform $new to XML
+
 ```
 {{toXML (mapJSON $new) -}}
 ```
 
 ### CSV to text
+
 - command
+
 ```sh
 bafi.exe -i users.csv -t myTemplate.tmpl -o output.txt
 ```
+
 users.csv
+
 ```
 name,surname
 John,"Jack Doe"
 ```
+
 - myTemplate.tmpl
+
 ```
 Users:
 {{- range .}}
 Name: {{.name}}, Surname: {{.surname}}
 {{- end}}
 ```
+
 note: CSV file must be **[RFC4180](https://datatracker.ietf.org/doc/html/rfc4180)** compliant, file must have header line and separator must be **comma ( , )**. Or you can use command line argument -d ( e.g. **-d ';'** or **-d 0x09** ) to define separator(delimiter).
 
 ### mt940 to CSV
+
 - mt940 returns simple struct (Header,Fields,[]Transactions) of strings and additional parsing needs to be done in template. This allows full flexibility on data processing
-- Identifiers are prefixed by **"F_"** (e.g. **:20:** = **.Fields.F_20**)
-- if parameter -d (delimiter e.g. -d "-\}\r\n" or "\r\n$") is defined for files with multiple messages (e.g. - Multicash), app returns array of mt940 messages. 
+- Identifiers are prefixed by **"F\_"** (e.g. **:20:** = **.Fields.F_20**)
+- if parameter -d (delimiter e.g. -d "-\}\r\n" or "\r\n$") is defined for files with multiple messages (e.g. - Multicash), app returns array of mt940 messages.
 - Note: This is actually good place to use integrated [LUA interpreter](/bafi/#lua-custom-functions) where you can create your own set of custom functions to parse data and easily reuse them in templates.
 
 - command
+
 ```sh
 bafi.exe -i message.sta -t myTemplate.tmpl -o output.csv
 ```
+
 - myTemplate.tmpl
+
 ```
-Reference, balance, VS 
+Reference, balance, VS
 {{- $F20 := .Fields.F_20 }}{{ $F60F := .Fields.F_60F }}
-{{range .Transactions }} 
+{{range .Transactions }}
 {{- $vsS := add (indexOf .F_86 "?21") 3 }} {{- $vsE := add $vsS 17 -}}
 {{- $F20}}, {{$F60F}}, {{slice .F_86 $vsS $vsE}}
 {{ end }}
 ```
 
 ### Any SQL to XML
+
 Bafi can be used in combination with very interesting tool **USQL** [https://github.com/xo/usql](https://github.com/xo/usql). USQL allows query almost any SQL like database (MSSQL,MySQL,postgres, ...) and get result in various formats. In this example we use -J for JSON. Output can be further processed by BaFi and templates
 
 ```sh
@@ -178,11 +230,15 @@ usql.exe mssql://user:password@server/instance/database -c "SELECT * FROM USERS"
 ```
 
 ### MongoDump to CSV
+
 - command
+
 ```sh
 bafi.exe -i users.bson -t myTemplate.tmpl -o output.html
 ```
+
 - myTemplate.tmpl
+
 ```
 name,surname
 {{- range .}}
@@ -191,17 +247,23 @@ name,surname
 ```
 
 ### Dashes in key names
+
 If key name contains dashes ( - ) bafi will fail with error "bad character U+002D '-'" for example:
+
 ```
 {{.my-key.subkey}}
 ```
+
 This is known limitation of go templates which can be solved by workaround
+
 ```
 {{index . "my-key" "subkey"}}
 ```
 
 ### Input autoformat to XXX
-Input data can be easily fomated to oher formats by functions **toXML,toJSON,toBSON,toYAML**. In this case its not necesarry add template file because it's as easy as 
+
+Input data can be easily fomated to oher formats by functions **toXML,toJSON,toBSON,toYAML**. In this case its not necesarry add template file because it's as easy as
+
 ```sh
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}" -o output.xml
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toJSON .}}" -o output.json
@@ -209,18 +271,31 @@ curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toB
 curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toYAML .}}" -o output.yml
 ```
 
+### ChatGPT query
+
+```sh
+curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "What's the current CZK rate?"
+curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "format rates to html" -gm gpt4
+./bafi -i invoice.json -gk myChatGPTToken -gq "create XML UBL format invoice" -o invoice.xml
+
+```
+
 ### Multiple input files
+
 Bafi can read multiple input files and merge them into one output file. This will require aditional file with files description.
 Description file must be in YAML format as described below and prefixed by question mark **"?"** for examle **bafi.exe -i ?files.yaml**
 
-Example: 
+Example:
 
 - batch file which gets the data from multiple sources **myFiles.bat**
+
 ```sh
 curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml > ecbRates.xml
 curl -s https://goweather.herokuapp.com/weather/prague > pragueWeather.json
 ```
+
 - Files description **myFiles.yaml**
+
 ```yaml
 - file: ./ecbRates.xml # file path
   format: xml # File format
@@ -229,7 +304,9 @@ curl -s https://goweather.herokuapp.com/weather/prague > pragueWeather.json
   format: json
   label: WEATHER
 ```
+
 - Template file **myTemplate.tmpl** which will generate simple HTML page with data
+
 ```html
 
     
@@ -239,7 +316,7 @@ curl -s https://goweather.herokuapp.com/weather/prague > pragueWeather.json
     

ECB Exchange rates from: {{dateFormat (index .RATES.Envelope.Cube.Cube "-time") "2006-01-02" "02.01.2006" }}

- {{- range .RATES.Envelope.Cube.Cube.Cube }} + {{- range .RATES.Envelope.Cube.Cube.Cube }} {{- end}}
currencyrate
{{index . "-currency" }}{{index . "-rate" }}
@@ -250,9 +327,9 @@ curl -s https://goweather.herokuapp.com/weather/prague > pragueWeather.json table, th, td { border: 1px solid black; width: 400px; } ``` + - Finally run bafi + ```sh bafi.exe -t myTemplate.tmpl -i ?myFiles.yaml -o output.html ``` - - diff --git a/go.mod b/go.mod index 4c2b782..b5fe506 100644 --- a/go.mod +++ b/go.mod @@ -12,3 +12,5 @@ require ( go.mongodb.org/mongo-driver v1.13.1 gopkg.in/yaml.v3 v3.0.1 ) + +require github.com/sashabaranov/go-openai v1.20.2 diff --git a/go.sum b/go.sum index 302730a..207b37e 100644 --- a/go.sum +++ b/go.sum @@ -15,13 +15,13 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mmalcek/mt940 v0.1.0 h1:MkjAGV1u2A6pa4z12YN1UdLCrhhNUpKRTrF/p5HVfdc= -github.com/mmalcek/mt940 v0.1.0/go.mod h1:IzQU3xpykKw6QEHn0i75Xxds7eapEEmwYn5L4B28ZZ8= github.com/mmalcek/mt940 v0.1.1 h1:w0LYJk4nQWnMeTtuLL1dY2oNMdtHTnWNVY+y7k0KMQU= github.com/mmalcek/mt940 v0.1.1/go.mod h1:IzQU3xpykKw6QEHn0i75Xxds7eapEEmwYn5L4B28ZZ8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sashabaranov/go-openai v1.20.2 h1:nilzF2EKzaHyK4Rk2Dbu/aJEZbtIvskDIXvfS4yx+6M= +github.com/sashabaranov/go-openai v1.20.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= diff --git a/main.go b/main.go index e72a075..5cd1b22 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/csv" "encoding/hex" "encoding/json" @@ -18,12 +19,13 @@ import ( "github.com/clbanning/mxj/v2" "github.com/mmalcek/mt940" + "github.com/sashabaranov/go-openai" lua "github.com/yuin/gopher-lua" "go.mongodb.org/mongo-driver/bson" "gopkg.in/yaml.v3" ) -const version = "1.1.2" +const version = "1.2.0" var ( luaData *lua.LState @@ -36,6 +38,9 @@ type tParams struct { inputFormat *string inputDelimiter *string getVersion *bool + chatGPTkey *string + chatGPTmodel *string + chatGPTquery *string } func init() { @@ -60,6 +65,9 @@ func main() { inputFormat: flag.String("f", "", "input format: json, bson, yaml, csv, mt940, xml(default)"), inputDelimiter: flag.String("d", "", "input delimiter: CSV only, default is comma -d ';' or -d 0x09"), getVersion: flag.Bool("v", false, "show version (Project page: https://github.com/mmalcek/bafi)"), + chatGPTkey: flag.String("gk", "", "OpenAI API key"), + chatGPTmodel: flag.String("gm", "gpt35", "OpenAI GPT-3 model (gpt35, gpt4)"), + chatGPTquery: flag.String("gq", "", "OpenAI query"), } flag.Parse() @@ -76,7 +84,7 @@ func processTemplate(params tParams) error { fmt.Printf("Version: %s\r\nProject page: https://github.com/mmalcek/bafi\r\n", version) return nil } - if *params.textTemplate == "" { + if *params.textTemplate == "" && *params.chatGPTkey == "" { fmt.Println("template file must be defined: -t template.tmpl") return nil } @@ -124,6 +132,29 @@ func processTemplate(params tParams) error { return err } } + + if *params.chatGPTkey != "" { + if *params.chatGPTquery == "" { + fmt.Println("OpenAI query must be defined: -gq \"What is the weather like?\"") + return nil + } + response, err := chatGPTprocess(mapData, params) + if err != nil { + return err + } + if *params.outputFile == "" { + fmt.Println(response.Choices[0].Message.Content) + } else { + output, err := os.Create(*params.outputFile) + if err != nil { + return fmt.Errorf("createOutputFile: %s", err.Error()) + } + defer output.Close() + output.WriteString(response.Choices[0].Message.Content) + } + return nil + } + templateFile, err := readTemplate(*params.textTemplate) if err != nil { return err @@ -316,6 +347,36 @@ func writeOutputData(mapData interface{}, outputFile *string, templateFile []byt return nil } +func chatGPTprocess(mapData interface{}, params tParams) (response openai.ChatCompletionResponse, err error) { + jsonData, err := json.Marshal(mapData) + if err != nil { + return response, fmt.Errorf("jsonMarshal: %s", err.Error()) + } + model := openai.GPT3Dot5Turbo + switch *params.chatGPTmodel { + case "gpt35": + model = openai.GPT3Dot5Turbo + case "gpt4": + model = openai.GPT4 + default: + model = openai.GPT3Dot5Turbo + } + + client := openai.NewClient(*params.chatGPTkey) + return client.CreateChatCompletion( + context.Background(), + openai.ChatCompletionRequest{ + Model: model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleUser, + Content: *params.chatGPTquery + "\n" + string(jsonData), + }, + }, + }, + ) +} + // cleanBOM remove UTF-8 Byte Order Mark if present func cleanBOM(b []byte) []byte { if len(b) >= 3 && diff --git a/main_test.go b/main_test.go index 159d5d3..230515b 100644 --- a/main_test.go +++ b/main_test.go @@ -17,6 +17,10 @@ func TestProcessTemplate(t *testing.T) { outputFile := "" textTemplate := `?{{define content}}` getVersion := false + chatGPTkey := "" + chatGPTmodel := "" + chatGPTquery := "" + params := tParams{ inputFile: &inputFile, inputFormat: &inputFormat, @@ -24,6 +28,9 @@ func TestProcessTemplate(t *testing.T) { outputFile: &outputFile, textTemplate: &textTemplate, getVersion: &getVersion, + chatGPTkey: &chatGPTkey, + chatGPTmodel: &chatGPTmodel, + chatGPTquery: &chatGPTquery, } err := processTemplate(params) if !strings.Contains(err.Error(), "stdin: Error-noPipe") {