Automated Model Card Generation V1
Abstract
Using the JSON schema from the Proposed JSON schema for BCI model cards – v0.0.0.1 , I used GPT4 via the OpenAI API to fill in the template for the p300_offline_test.py example within BessyPython.
The inputs were:
- The
p300_offline_test.pyfile – the “experiment main file” - Stack trace generated from running
p300_offline_test.py - (Optionally) The relevant code files from
bci-essentials-python– namely: erp_data.pyeeg_data.pysignal_processing.pyclassification.erp_rg_classifier.py
Using the API I was able to control the model temperature, and tested 3 values: 0.3, 0.5, 0.7. The lower values are more “deterministic and focused” and the higher values are “more creative”. The value used by ChatGPT is 0.7.
Additionally, the API allows me to set a “seed” value, which should help get more repeatable results from GPT4 compared to when using the “Chat” interface. I chose 42 for the seed.
Quick Results
Using API allows more controlled and reproducible outputs from GPT4 for filling in the model card template, with best result using all 3 inputs and a lower model temperature.
Next steps
Evaluate how well this approach works with the other example files in the bessyPython library.
Longer Results
The best results so far are with the lower temperature (0.3) and using all 3 inputs (the “experiment main file”, the stack trace and the relevant code from bci-essentials-python). The cost for this request was ~$0.30, while using a “parsed” version of the stack trace. If it used a longer, less filtered, version of the stack trace the cost would be ~$0.50.
{
"BCI Model card version": "0.0.0.1",
"Pipeline Name": "P300 Offline Test",
"Pipeline Steps": [
{
"Step": "Generic",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Resampling",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Preprocessing",
"Used": true,
"From": "bci_essentials.signal_processing",
"Function": "bandpass",
"Description": "Bandpass filter applied to EEG data",
"Details": "Filters out frequencies outside of the range f_low to f_high",
"parameters": {
"f_low": {
"type": "float",
"description": "Lower corner frequency",
"value": 0.1
},
"f_high": {
"type": "float",
"description": "Upper corner frequency",
"value": 10
},
"order": {
"type": "int",
"description": "Order of the filter",
"value": 5
},
"f_sample": {
"type": "float",
"description": "Sampling rate of signal",
"value": 256
}
}
},
{
"Step": "Feature selection",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Channel selection",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Spatial filtering",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Classification",
"Used": true,
"From": "bci_essentials.classification.erp_rg_classifier",
"Function": "ErpRgClassifier",
"Description": "ERP RG Classifier for classifying ERPs using Riemannian Geometry",
"Details": "Uses XdawnCovariances, TangentSpace, and LinearDiscriminantAnalysis from pyriemann and sklearn",
"parameters": {
"Model architecture": {
"param_1": {
"type": "string",
"description": "Covariance estimator",
"value": "oas"
}
},
"Hyperparameters": {
"n_splits": {
"type": "int",
"description": "Number of folds for cross-validation",
"value": 5
},
"lico_expansion_factor": {
"type": "int",
"description": "Linear Combination Oversampling expansion factor",
"value": 4
},
"oversample_ratio": {
"type": "float",
"description": "Traditional oversampling ratio",
"value": 0
},
"undersample_ratio": {
"type": "float",
"description": "Traditional undersampling ratio",
"value": 0
},
"random_seed": {
"type": "int",
"description": "Random seed",
"value": 35
}
},
"Training parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
}
}
],
"Individual Fields": {
"name": "Participant Name",
"channels selection": {
"channels rejected": null,
"channels selected": null
},
"classifier performance": {
"accuracy": null,
"AUC": null,
"precision": null,
"recall": null
}
},
"Population Fields": {
"name": "Participant Name",
"Population database": null,
"Population metrics": {
"accuracy": null,
"AUC": null,
"precision": null,
"recall": null
}
}
}
While using just the “experiment main” file and the stack trace also worked well, it sometimes said that the “Preprocessing” step was coming from method bci_essentials.erp_data.setup(), rather than bci_essentials.signal_processing. The cost of this ranged from ~$0.10 to ~$0.30, depending on whether the longer or shorter version of the stack trace was used.
{
"BCI Model card version": "0.0.0.1",
"Pipeline Name": "P300 Offline Test",
"Pipeline Steps": [
{
"Step": "Generic",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Resampling",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Preprocessing",
"Used": true,
"From": "bci_essentials.erp_data",
"Function": "setup",
"Description": "Preprocessing of EEG data for P300 ERP",
"Details": "Bandpass filtering is applied as part of preprocessing",
"parameters": {
"pp_low": {
"type": "float",
"description": "Lower corner frequency for bandpass filter",
"value": 0.1
},
"pp_high": {
"type": "float",
"description": "Upper corner frequency for bandpass filter",
"value": 10
},
"pp_order": {
"type": "int",
"description": "Order of the bandpass filter",
"value": 5
},
"fsample": {
"type": "float",
"description": "Sampling rate of the EEG signal",
"value": 256.0
}
}
},
{
"Step": "Feature selection",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Channel selection",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Spatial filtering",
"Used": false,
"From": null,
"Function": null,
"Description": null,
"Details": null,
"parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
},
{
"Step": "Classification",
"Used": true,
"From": "bci_essentials.classification.erp_rg_classifier",
"Function": "ErpRgClassifier",
"Description": "Classification of ERP signals for P300 detection",
"Details": "ERP classifier with settings for P300",
"parameters": {
"Model architecture": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
},
"Hyperparameters": {
"n_splits": {
"type": "int",
"description": "Number of folds for cross-validation",
"value": 5
},
"covariance_estimator": {
"type": "string",
"description": "Covariance estimator. See pyriemann Covariances.",
"value": "oas"
},
"lico_expansion_factor": {
"type": "int",
"description": "Lico expansion factor",
"value": 4
},
"oversample_ratio": {
"type": "int",
"description": "Oversample ratio for balancing classes",
"value": 0
},
"undersample_ratio": {
"type": "int",
"description": "Undersample ratio for balancing classes",
"value": 0
},
"random_seed": {
"type": "int",
"description": "Random seed for reproducibility",
"value": 35
}
},
"Training parameters": {
"param_1": {
"type": null,
"description": null,
"value": null
},
"param_2": {
"type": null,
"description": null,
"value": null
},
"param_N": {
"type": null,
"description": null,
"value": null
}
}
}
}
],
"Individual Fields": {
"name": "Participant Name",
"channels selection": {
"channels rejected": null,
"channels selected": null
},
"classifier performance": {
"accuracy": null,
"AUC": null,
"precision": null,
"recall": null
}
},
"Population Fields": {
"name": "Participant Name",
"Population database": null,
"Population metrics": {
"accuracy": null,
"AUC": null,
"precision": null,
"recall": null
}
}
}