sailing-analytics/java/com.sap.sailing.windestimation.lab/python/Maneuver data analysis.ipynb

Feature selection manoeuvre-based wind estimation

First the MongoDB loading and pandas configuration code

In [ ]:

!pip install pymongo

Initialiaze pandas and co.

In [ ]:

%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from pymongo import MongoClient
from sklearn.feature_selection import SelectPercentile
import pathlib
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import seaborn as sns
from sklearn.feature_selection import chi2
from sklearn.feature_selection import f_classif

In [ ]:

def _connect_mongo(host, port, username, password, db):
    if username and password:
        mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)
        conn = MongoClient(mongo_uri)
    else:
        conn = MongoClient(host, port)
    return conn[db]
def read_mongo(db, collection, query={}, host='localhost', port=27017, username=None, password=None, no_id=False):
    db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)
    cursor = db[collection].find(query)
    df = pd.DataFrame(list(cursor))
    if no_id:
        del df['_id']
    return df
def init():
    desired_width = 320
    pd.set_option('display.width', desired_width)
    np.set_printoptions(linewidth=desired_width)
    pd.set_option("display.max_columns", desired_width)
    pd.set_option("display.max_info_rows", 10000000)
    plt.figure.max_open_warning = 1000
    sns.set_style('whitegrid')

In [ ]:

init()

Load manoeuvre data ignoring irrelevant manoeuvre data

In [ ]:

maneuvers = read_mongo('windEstimation', 'maneuversForDataAnalysis', {'category': {'$ne': 'SMALL'}, 'clean': True, "deviationTackAngle":{"$ne":None}, "deviationJibeAngle":{"$ne":None}, 'nextMarkBefore':{"$ne":None}, 'nextMarkAfter':{"$ne":None}})
maneuvers.info()

Convert categorical boat hull type column to multiple binary columns

In [ ]:

hullType = pd.get_dummies(maneuvers['hullType'], drop_first=True)
maneuvers.drop(['hullType'],axis=1,inplace=True)
maneuvers = pd.concat([maneuvers, hullType],axis=1)
maneuvers.head()

Split manoeuvre data by categories

In [ ]:

_360 = maneuvers[maneuvers['category'] == '_360']
_180 = maneuvers[maneuvers['category'] == '_180']
wide = maneuvers[maneuvers['category'] == 'WIDE']
regularWithMP = maneuvers[(maneuvers['category'] == 'REGULAR') | (maneuvers['category'] == 'MARK_PASSING')]
regularWithoutMP = maneuvers[maneuvers['category'] == 'REGULAR']
contenderRegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'Contender']
j24RegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'J/24']
formula18RegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'Formula 18']
contenderRegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'Contender']
j24RegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'J/24']
formula18RegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'Formula 18']
plt.figure()
plt.bar(['Regular/MP', 'Regular', 'Wide', '180', '360'], [regularWithMP['_id'].count(), regularWithoutMP['_id'].count(), wide['_id'].count(), _180['_id'].count(), _360['_id'].count()])
plt.title('Maneuvers of all boat classes')
plt.figure()
plt.bar(['Contender', 'J/24', 'Formula 18'], [contenderRegularWithMP['_id'].count(), j24RegularWithMP['_id'].count(), formula18RegularWithMP['_id'].count()])
plt.title('Regular WITH mark passing maneuvers of chosen boat classes')
plt.figure()
plt.bar(['Contender', 'J/24', 'Formula 18'], [contenderRegularWithoutMP['_id'].count(), j24RegularWithoutMP['_id'].count(), formula18RegularWithoutMP['_id'].count()])
plt.title('Regular WITHOUT mark passing maneuvers of chosen boat classes')
targetColumn = 'type'
maneuverTypes = ['TACK', 'JIBE', 'HEAD_UP', 'BEAR_AWAY', 'OTHER', '_180_TACK', '_180_JIBE', '_360']

In [ ]:

_360['_id'].count()

Prepare datasets for automatic iterations, e.g. for plotting

In [ ]:

datasets = [regularWithMP, regularWithoutMP, contenderRegularWithMP, j24RegularWithMP, formula18RegularWithMP, contenderRegularWithoutMP, j24RegularWithoutMP, formula18RegularWithoutMP, wide, _180, _360]
datasetsTitle = ['Regular/MP', 'Regular', 'Contender RMP', 'J/24 RMP', 'Formula 18 RMP', 'Contender R', 'J/24 R', 'Formula 18 R', 'Wide', '180', '360']
def splitDatasetsByType():
    datasetsSplitByType = []
    datasetsSplitByTypeLegend = []
    for dataset in datasets:
        splitDatasets = {}
        for maneuverType, subDataset in dataset.groupby(targetColumn):
            splitDatasets[maneuverType] = subDataset
        splitDatasetsSorted = []
        splitDatasetsSortedLegend = []
        for maneuverType in maneuverTypes:
            if maneuverType in splitDatasets:
                splitDatasetsSorted.append(splitDatasets[maneuverType])
                splitDatasetsSortedLegend.append(maneuverType.replace('_', ''))
        finalSplitDatasetsSortedLegend = []
        datasetsSplitByType.append(splitDatasetsSorted)
        datasetsSplitByTypeLegend.append(splitDatasetsSortedLegend)
    return (datasetsSplitByType, datasetsSplitByTypeLegend)
datasetsSplitByType, datasetsSplitByTypeLegend = splitDatasetsByType()

In [ ]:

def plotCounts(dataset, legend, datasetTitle):
    plt.figure()
    plt.bar(legend, [subDataset['maxTurningRate'].count() for subDataset in dataset])
    plt.title(datasetTitle)

In [ ]:

i = 0
for dataset in datasets:
    plotCounts(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], datasetsTitle[i])
    i += 1

In [ ]:

_180 = _180[(_180[targetColumn] == '_180_TACK') | (_180[targetColumn] == '_180_JIBE')]

In [ ]:

sns.countplot(x = targetColumn, data = _180, palette = 'rainbow')

Plot all numeric attributes in each dataset as box plot and histogram

In [ ]:

def boxplot(dataset, legend, attribute, datasetTitle, savePath=None):
    fig = plt.figure()
    if attribute.startswith('twa'):
        dataset = [series[attribute].abs() for series in dataset]
    else:
        dataset = [series[attribute] for series in dataset]
    plt.boxplot(dataset, showfliers=False)
    newLegend = []
    for i, legendItem in enumerate(legend, 1):
        newLegend.append(str(i) + ' - ' + legendItem)
    plt.legend(newLegend)
    plt.title(datasetTitle + ' - box plot of ' + attribute)
    if savePath is not None:
        fig.savefig(savePath)
        plt.close(fig)
def hist(dataset, legend, attribute, datasetTitle, savePath=None):
    fig = plt.figure()
    minX = 1
    maxX = 0
    for series in dataset:
        minCandidate = series[attribute].quantile(0.01)
        maxCandidate = series[attribute].quantile(0.99)
        minX = min(minX, minCandidate)
        maxX = max(maxX, maxCandidate)
    dataset = [series[attribute] for series in dataset]
    plt.hist(dataset, density=True, histtype='bar', bins=20, range=[minX, maxX])
    plt.legend(legend)
    plt.title(datasetTitle + ' - histogram of ' + attribute)
    if savePath is not None:
        fig.savefig(savePath)
        plt.close(fig)

In [ ]:

def visualizeData(imgFolderName):
    pathlib.Path(imgFolderName).mkdir(parents=True, exist_ok=True)
    for attribute in list(maneuvers):
        print('## ' + attribute)
        i = 0
        for dataset in datasets:
            if np.issubdtype(dataset[attribute].dtype, np.number):
                print('##### ' + datasetsTitle[i])
                dataset.describe()
                boxplot(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], attribute, datasetsTitle[i], imgFolderName + '/' + attribute + '_boxplot_' + str(i))
                hist(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], attribute, datasetsTitle[i], imgFolderName + '/' + attribute + '_hist_' + str(i))
                i += 1
    

In [ ]:

visualizeData('dataVisualisation1')

Analyse wide maneuvers

In [ ]:

sns.countplot(x = targetColumn, data = wide, palette = 'rainbow')

In [ ]:

sns.countplot(x = targetColumn, data = _180, palette = 'rainbow')

In [ ]:

sns.countplot(x = targetColumn, data = _360, palette = 'rainbow')

Check TWAs at middle course, lowest speed, max turning rate for wide and circular maneuvers

In [ ]:

def plotTwas():
    sns.set_style('whitegrid')
    twaFeatures = ['twaMiddleCourse', 'twaMiddleCourseMainCurve', 'twaLowestSpeed', 'twaMaxTurnRate']
    twaDatasets = [wide[wide[targetColumn] == '_180_TACK'], wide[wide[targetColumn] == '_180_JIBE'], _360[_360[targetColumn] == '_360']]
    twaDatasetsTitle = ['Wide tacks', 'Wide jibes', 'Penalty circles']
    i = 0
    for dataset in twaDatasets:
        for twaFeature in twaFeatures:
            data = [dataset[dataset['starboard'] == False], dataset[dataset['starboard'] == True]]
            title = twaDatasetsTitle[i] + " - " + twaFeature
            hist(data, ['port', 'starboard'], twaFeature, title)
        i += 1

In [ ]:

plotTwas()

Feature selection

In [ ]:

irrelevantFeatures = ['type', 'trackId', 'category', '_id', 'clean', 'twaBefore', 'twaAfter', 'tws', 'boatClass', 'fixesCountForPolars', 'twaMiddleCourse', 'twaMiddleCourseMainCurve', 'twaLowestSpeed', 'twaMaxTurnRate', 'starboard', 'hullLength', 'hullBeam', 'SURFERBOARD', 'MONOHULL', 'speedBefore', 'speedAfter']
def getInputOutputVectors(maneuvers):
    inputVector = maneuvers.drop(irrelevantFeatures, axis=1)
    outputVector = maneuvers['type']
    inputVectorPositive = inputVector.drop(['deviationJibeAngle', 'deviationTackAngle', 'timeLoss'], axis=1)
    inputVectorPositive['absDeviationJibeAngle'] = inputVector['deviationJibeAngle'].abs()
    inputVectorPositive['absDeviationTackAngle'] = inputVector['deviationTackAngle'].abs()
    inputVectorPositive['absTimeLoss'] = inputVector['timeLoss'].abs()
    return (inputVector, inputVectorPositive, outputVector)

In [ ]:

from collections import OrderedDict
def featureSelectionPlot(maneuvers, scoreFunc, onlyPositive, title):
    print('#####' + title)
    inputV, inputVPositive, outputV = getInputOutputVectors(maneuvers)
    if onlyPositive:
        inputV = inputVPositive
    test =  SelectPercentile(score_func=scoreFunc, percentile=5)
    fit = test.fit(inputV, outputV)
    np.set_printoptions(precision=3)
    tempScores = {}
    for i in range(len(inputV.columns)):
        tempScores[inputV.columns[i]] = fit.scores_[i] / max(fit.scores_)
    i = 1
    sortedScores = OrderedDict()
    for key, value in sorted(tempScores.items(), reverse=True, key=lambda kv: kv[1]):
        print(str(i) + '. ' + key + ":\t" + str(value))
        sortedScores[key] = value
        i += 1
    plt.figure(figsize = (6,6))
    plt.barh(list(sortedScores.keys())[::-1], list(sortedScores.values())[::-1])
    plt.title(title)

Feature selection with mutual_info_classif

In [ ]:

def testFeatures(score_func, onlyPositive):
    i = 0
    for dataset in datasets:
        featureSelectionPlot(dataset, score_func, onlyPositive, datasetsTitle[i])
        i += 1

In [ ]:

testFeatures(f_classif, False)

Feature selection with chi2 (is a wrong choice for the given data because it is meant to be used for categorical input features)

In [ ]:

#testFeatures(chi2, True)

In [ ]:

def testFeaturesWithTree():
    i = 0
    for dataset in datasets:
        plotFeatureSelectionWithTree(datasets[i], datasetsTitle[i])
        i += 1
def plotFeatureSelectionWithTree(maneuvers, title):
    print('#####' + title)
    inputV, inputVPositive, outputV = getInputOutputVectors(maneuvers)
    clf = RandomForestClassifier(n_estimators=50)
    clf = clf.fit(inputV, outputV)
    tempScores = {}
    for i in range(len(inputV.columns)):
        tempScores[inputV.columns[i]] = clf.feature_importances_[i]
    i = 1
    sortedScores = OrderedDict()
    for key, value in sorted(tempScores.items(), reverse=True, key=lambda kv: kv[1]):
        print(str(i) + '. ' + key + ":\t" + str(value))
        sortedScores[key] = value
        i += 1
    plt.figure(figsize=(6,6))
    plt.barh(list(sortedScores.keys())[::-1], list(sortedScores.values())[::-1])
    plt.title(title)

In [ ]:

testFeaturesWithTree()

Feature correlations

In [ ]:

i = 0
for dataset in datasets:
    plt.figure(figsize = (6,6))
    sns.heatmap(dataset.drop(irrelevantFeatures, axis=1).corr(),cmap='coolwarm')
    plt.title(datasetsTitle[i])
    i += 1

Check feature individual feature correlations

In [ ]:

sns.jointplot(x='speedLossRatio',y='lowestVsExitingSpeedRatio',data=regularWithMP)
plt.title("Speed loss vs. lowestSpeedVsExitingSpeed in Regular/MP")

In [ ]:

sns.jointplot(x='speedLossRatio',y='maxTurningRate',data=regularWithMP)
plt.title("Speed loss vs. maxTurningRate in Regular/MP")

Head-up and Bear-away maneuvers are very similar. => Merge head ups and bear aways

In [ ]:

def mergeHeadUpsAndBearAways(maneuvers):
    maneuvers['type'] = maneuvers['type'].apply([lambda mType: 'OTHER' if mType == 'BEAR_AWAY' or mType == 'HEAD_UP' else mType])

In [ ]:

for dataset in datasets:
    mergeHeadUpsAndBearAways(dataset)
    dataset[targetColumn].unique()
datasetsSplitByType, datasetsSplitByTypeLegend = splitDatasetsByType()
i = 0
for dataset in datasets:
    plotCounts(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], datasetsTitle[i])
    i += 1

Visualize data with merged head ups and bear aways

In [ ]:

visualizeData('dataVisualisation2')

Remove attributes with low relevance

In [ ]:

finalIrrelevantFeatures = irrelevantFeatures + ['timeLoss', 'speedInOutRatio', 'recoveryPhaseDuration', 'oversteering', 'maneuverDuration', 'mainCurveDuration', 'absMainCurveAngle', 'lowestVsExitingSpeedRatio']
finalIrrelevantFeatures

Construct final datasets

Define datasets for evaluation

In [ ]:

polarFeatures = ['deviationTackAngle', 'deviationJibeAngle']
markFeatures = ['markPassing']
def dropFeatures(dataset, features = []):
    return dataset.drop(features + finalIrrelevantFeatures[2:], axis=1)
evaluationDatasets = [dropFeatures(regularWithMP), dropFeatures(regularWithMP, polarFeatures), dropFeatures(regularWithMP, markFeatures), dropFeatures(regularWithMP, markFeatures + polarFeatures), dropFeatures(contenderRegularWithMP), dropFeatures(j24RegularWithMP), dropFeatures(formula18RegularWithMP), dropFeatures(contenderRegularWithMP, markFeatures), dropFeatures(j24RegularWithMP, markFeatures), dropFeatures(formula18RegularWithMP, markFeatures), dropFeatures(wide, markFeatures + polarFeatures), dropFeatures(_180, markFeatures + polarFeatures)]
evaluationDatasetsTitle = ['PolarsMarks', 'Marks', 'Polars', 'Basic', 'Contender Marks', 'J/24 Marks', 'Formula 18 Marks', 'Contender Basic', 'J/24 Basic', 'Formula 18 Basic', 'Wide', '180']
i = 0
for dataset in evaluationDatasets:
    print(evaluationDatasetsTitle[i])
    print(dataset.columns)
    i += 1

Split datasets into training set and test set. The test set shall contain all manoeuvres recorded within year 2015. All other manoeuvres shall be used for training.

In [ ]:

def splitTrainTest(evaluationDatasets, dropFeatures=[]):
    trainDatasetsInput = []
    trainDatasetsOutput = []
    testDatasetsInput = []
    testDatasetsOutput = []
    i = 0
    for dataset in evaluationDatasets:
        tempDataset = dataset[[targetColumn]]
        tempDataset['trainTest'] = dataset['trackId'].apply(lambda trackId: 'TEST' if '2018' in trackId else 'TRAIN')
        trainDataset = dataset[~dataset['trackId'].str.contains("2018")]
        testDataset = dataset[dataset['trackId'].str.contains("2018")]
        trainDatasetsInput.append(trainDataset.drop(['trackId', targetColumn] + dropFeatures, axis=1))
        testDatasetsInput.append(testDataset.drop(['trackId', targetColumn] + dropFeatures, axis=1))
        trainDatasetsOutput.append(trainDataset[targetColumn])
        testDatasetsOutput.append(testDataset[targetColumn])
        plotCounts([trainDataset, testDataset], ['Train', 'Test'], 'Train/test split for ' + evaluationDatasetsTitle[i])
        plt.figure()
        sns.countplot(x = targetColumn, hue = 'trainTest', data = tempDataset, palette = 'rainbow')
        plt.title(evaluationDatasetsTitle[i])
        i += 1
    return (trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput)

In [ ]:

(trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput) = splitTrainTest(evaluationDatasets)

In [ ]:

(trainDatasetsOutput[0] == 'TACK').sum()

In [ ]:

i = 0
for dataset in trainDatasetsInput:
    print('# ' + evaluationDatasetsTitle[i])
    print(dataset.columns)
    print('#')
    i += 1

Determine optimal number of components for PCA

In [ ]:

def determineOptimalNumberOfPCAComponents(inputFeatures, label=None):
    scaler = StandardScaler()
    scaledInputFeatures = scaler.fit_transform(inputFeatures)
    pca = PCA()
    pca.fit(scaledInputFeatures)
    cumsum = np.cumsum(pca.explained_variance_ratio_)
    d = np.argmax(cumsum >= 0.95) + 1
    if label:
        print(label + ":\t" + str(d) + '/' + str(len(inputFeatures.columns)))
    return d

In [ ]:

i = 0
for inputDataset in trainDatasetsInput:
    determineOptimalNumberOfPCAComponents(inputDataset, evaluationDatasetsTitle[i])
    i += 1

Feature scaling and PCA

In [ ]:

def transformInput(inputFeatures):
    d = determineOptimalNumberOfPCAComponents(inputFeatures)
    pipeline = Pipeline([('scaling', StandardScaler()), ('pca', PCA(n_components=d))])
    transformedInputFeatures = pipeline.fit_transform(inputFeatures)
    return transformedInputFeatures

In [ ]:

scaledTrainDatasetsInput = []
for inputDataset in trainDatasetsInput:
    scaledTrainDatasetsInput.append(StandardScaler().fit_transform(inputDataset))

In [ ]:

transformedTrainDatasetsInput = []
for inputDataset in trainDatasetsInput:
    transformedTrainDatasetsInput.append(transformInput(inputDataset))

Evaluate various ML models

In [ ]:

def determineBestScoreAndParams(inputFeatures, outputFeatures, clf, paramGrid, label=None):
    gridSearch = GridSearchCV(clf, paramGrid, cv=3,
                               scoring='f1_macro', return_train_score=True, n_jobs=4)
    gridSearch.fit(inputFeatures, outputFeatures)
    if label:
        print(label + " - score:\t" + str(gridSearch.best_score_))
        print(str(gridSearch.cv_results_))
        print('Best params:')
        print(str(gridSearch.best_params_))
        print('###########################')
    else:
        return gridSearch
def crossValidation(clf, inputFeatures=trainDatasetsInput[0], outputFeatures=trainDatasetsOutput[0], label=None):
    res = cross_val_score(clf, inputFeatures, outputFeatures, cv=3, n_jobs=4, scoring='f1_macro')
    if label:
        print(label + ":\t" + str(res))
    return res

In [ ]:

clf = RandomForestClassifier(n_estimators=100)
crossValidation(clf)

In [ ]:

clf = KNeighborsClassifier(n_neighbors=20)
crossValidation(clf)

In [ ]:

clf = GaussianNB() 
crossValidation(clf)

In [ ]:

clf = BernoulliNB() 
crossValidation(clf)

In [ ]:

clf = LinearSVC(dual=False, C=1) 
crossValidation(clf)

In [ ]:

clf = LogisticRegression()
crossValidation(clf)

In [ ]:

clf = SGDClassifier(penalty='l2', max_iter=5, tol=None)
crossValidation(clf)

In [ ]:

clf = QuadraticDiscriminantAnalysis()
crossValidation(clf)

In [ ]:

clf = LinearDiscriminantAnalysis()
crossValidation(clf)

In [ ]:

clf = MLPClassifier(hidden_layer_sizes=(1000, 1000, 100), activation='relu')
crossValidation(clf)

Compare classifiers considering it with its assumed optimal hyperparameters (Too many hyperparameter combinations consume a lot of computation time)

In [ ]:

classifiersWithParamsGrid = [
    (GaussianNB(), [
        {}
    ]),
    (QuadraticDiscriminantAnalysis(), [
        {}
    ]),
    (LinearDiscriminantAnalysis(), [
        {}
        #{'solver': ['svd', 'lsqr', 'eigen']}
    ]),
    (GradientBoostingClassifier(), [
        {}
    ]),
    (LinearSVC(dual=False, C=1), [
        {}
        # {'C': [0.1, 0.5, 1, 10, 100]}
    ]),
    (LogisticRegression(), [
        {}
        # {'C': [10, 100, 200], 'max_iter': [20, 50, 100]}
    ]),
    #(SGDClassifier(tol=None), [
    #    {}
        # {'penalty': ['l1', 'l2', 'elasticnet'], 'max_iter': [3, 5, 10, 20]}
    #]),
    (RandomForestClassifier(n_estimators=30), [
        {}
        # {'n_estimators': [30], 'max_features': [4], 'max_depth': [None]}
    ]),
    (KNeighborsClassifier(n_neighbors=25), [
        {}
        # {'n_neighbors': [15, 20, 25, 30]}
    ]),
    (MLPClassifier(hidden_layer_sizes=(100, 100), activation='relu'), [
        {}
        # {'hidden_layer_sizes': [(100), (100, 100), (10, 10), (100, 10), (100, 100, 10)], 'activation': ['logistic', 'tanh', 'relu']}
    ])
]

Test classifiers on evaluation datasets

In [ ]:

def testClassifiers(classifiersWithParamsGrid, filterDatasetsWithMaxElements=None):
    testScores = []
    trainScores = []
    fitTimes = []
    scoreTimes = []
    clfNames = []
    bestParams = []
    for clfWithParamsGrid in classifiersWithParamsGrid:
        clf = clfWithParamsGrid[0]
        paramsGrid = clfWithParamsGrid[1]
        clfName = clf.__class__.__name__
        clfNames.append(clfName)
        clfTestScores = []
        clfTrainScores = []
        clfFitTimes = []
        clfScoreTimes = []
        clfBestParams = []
        testScores.append(clfTestScores)
        trainScores.append(clfTrainScores)
        fitTimes.append(clfFitTimes)
        scoreTimes.append(clfScoreTimes)
        bestParams.append(clfBestParams)
        print('########################')
        print('Classifier: ' + clfName)
        i = 0
        for normal, scaled, transformed in zip(trainDatasetsInput, scaledTrainDatasetsInput, transformedTrainDatasetsInput):
            outputFeatures = trainDatasetsOutput[i]
            if filterDatasetsWithMaxElements and len(outputFeatures) > filterDatasetsWithMaxElements:
                print('Skipping Dataset ' + evaluationDatasetsTitle[i] + ' with ' + str(len(outputFeatures)) + ' elements')
                i += 1
                continue
            print('Dataset ' + evaluationDatasetsTitle[i])
            for inputFeatures, inputFeaturesTitle in [(normal, 'Normal'), (scaled, 'Scaled'), (transformed, 'Transformed')]:
                print(inputFeaturesTitle)
                gridSearch = determineBestScoreAndParams(inputFeatures, outputFeatures, clf, paramsGrid)
                scores = gridSearch.cv_results_
                testScore = max(scores['mean_test_score'])
                j = np.argmax(scores['mean_test_score'])
                trainScore = scores['mean_train_score'][j]
                fitTime = scores['mean_fit_time'][j]
                scoreTime = scores['mean_score_time'][j]
                clfTestScores.append(testScore)
                clfTrainScores.append(trainScore)
                clfFitTimes.append(fitTime)
                clfScoreTimes.append(scoreTime)
                clfBestParams.append(gridSearch.best_params_)
                print("testScore:\t" + str(testScore))
                print("trainScore:\t" + str(trainScore))
                print("fitTime:\t" + str(fitTime))
                print("scoreTime:\t" + str(scoreTime))
                print("best_params_:\n" + str(gridSearch.best_params_))
                print("mean_test_scores:" + str(scores['mean_test_score']))
                print("mean_test_scores:" + str(scores['mean_train_score']))
                print("mean_fit_times:" + str(scores['mean_fit_time']))
                print("mean_score_times:" + str(scores['mean_score_time']))
            i += 1
    return (testScores, trainScores, fitTimes, scoreTimes, clfNames, bestParams)

In [ ]:

scores = testClassifiers(classifiersWithParamsGrid)

In [ ]:

(trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput) = splitTrainTest(evaluationDatasets, dropFeatures=['scaledSpeedBefore', 'scaledSpeedAfter'])
scaledTrainDatasetsInput = []
for inputDataset in trainDatasetsInput:
    scaledTrainDatasetsInput.append(StandardScaler().fit_transform(inputDataset))
transformedTrainDatasetsInput = []
for inputDataset in trainDatasetsInput:
    transformedTrainDatasetsInput.append(transformInput(inputDataset))

In [ ]:

i = 0
for dataset in trainDatasetsInput:
    print('# ' + evaluationDatasetsTitle[i])
    print(dataset.columns)
    print('#')
    i += 1

In [ ]:

scoresWithoutScaledSpeed = testClassifiers(classifiersWithParamsGrid)

Plot the classifier statistics

In [ ]:

def plotScores(scores, testScores, clfNames, label, scoringWithMax):
    print('########')
    print(label)
    tempScores = []
    sortedScores = []
    for i in range(int(len(testScores[0]) / 3)):
        tempScores.append({})
        sortedScores.append(OrderedDict())
    for clfName, clfIndex in zip(clfNames, range(len(clfNames))):
        for i in range(int(len(testScores[0]) / 3)):
            testScoresToCheck = []
            for k in range(2):
                testScoresToCheck.append(testScores[clfIndex][i * 3 + k])
            bestArg = np.argmax(testScoresToCheck) if scoringWithMax else np.argmin(testScoresToCheck)
            tempScores[i][clfName] = scores[clfIndex][i * 3 + bestArg]
    rev = True if scoringWithMax else False
    revIndex = -1 if scoringWithMax else 1
    for i in range(int(len(testScores[0]) / 3)):
        print('# ' + evaluationDatasetsTitle[i])
        num = 1
        for key, value in sorted(tempScores[i].items(), reverse=rev, key=lambda kv: kv[1]):
            print(str(num) + '. ' + key + ":\t" + str(value))
            sortedScores[i][key] = value
            num += 1
        fig = plt.figure()
        plt.barh(list(sortedScores[i].keys())[::revIndex], list(sortedScores[i].values())[::revIndex])
        plt.title(label + ' - ' + evaluationDatasetsTitle[i])
        #plt.close(fig)

In [ ]:

plotScores(scores[0], scores[0], scores[4], 'TestScore', True)

In [ ]:

plotScores(scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'TestScore', True)

In [ ]:

plotScores(scores[1], scores[0], scores[4], 'TrainScore', True)

In [ ]:

plotScores(scoresWithoutScaledSpeed[1], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'TrainScore', True)

In [ ]:

plotScores(scores[2], scores[0], scores[4], 'FitTimes', False)

In [ ]:

plotScores(scoresWithoutScaledSpeed[2], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'FitTimes', False)

In [ ]:

plotScores(scores[3], scores[0], scores[4], 'ScoreTimes', False)

In [ ]:

plotScores(scoresWithoutScaledSpeed[3], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'ScoreTimes', False)

Check classifiers performance on transformed, scaled and original datasets

In [ ]:

def plotTransformationScores(scores, testScores, clfNames, label, scoringWithMax):
    print('#################################')
    print(label)
    rev = True if scoringWithMax else False
    revIndex = -1 if scoringWithMax else 1
    legend = ['Normal', 'Scaled', 'Transformed']
    bestArgs = []
    tempScores = {}
    sortedScores = {}
    for clfName, clfIndex in zip(clfNames, range(len(clfNames))):
        tempScores[clfName] = []
        sortedScores[clfName] = []
        for i in range(int(len(testScores[0]) / 3)):
            tempScores[clfName].append({})
            sortedScores[clfName].append(OrderedDict())
            for k in range(3):
                tempScores[clfName][i][legend[k]] = scores[clfIndex][i * 3 + k]
    for clfName in clfNames:
        print('########### ' + clfName)
        for i in range(int(len(testScores[0]) / 3)):
            print('# ' + evaluationDatasetsTitle[i])
            num = 1
            for key, value in sorted(tempScores[clfName][i].items(), reverse=rev, key=lambda kv: kv[1]):
                print(str(num) + '. ' + key + ":\t" + str(value))
                sortedScores[clfName][i][key] = value
                num += 1
            fig = plt.figure()
            print(list(sortedScores[clfName][i].keys())[::revIndex])
            print('#')
            print(list(sortedScores[clfName][i].values())[::revIndex])
            plt.barh(list(sortedScores[clfName][i].keys())[::revIndex], list(sortedScores[clfName][i].values())[::revIndex])
            plt.title(label + ': ' + clfName + ' - ' + evaluationDatasetsTitle[i])
            # plt.close(fig)

Warning:

Output truncated. This notebook contains too many cells to display efficiently.