Python Pandas comparing datetime values in DataFrame cells

Question

I run two sets of speed tests and recorded the data into CSV files which I then read back in and converted into DataFrames. When I display the data it looks like this and I have 2 sets of it; one for test#1 and one for test#2

DataFrame results table example

What I'd like to do is to compare each cell of test#1 'Time Elapsed' column with corresponding cell of test#2 'Time Elapsed' column and in a new DataFrame display in percentages the change (i.e. +1.05% or -4.72%). I don't know how to access those cells and do any comparisons on them since they are weird data type?

To generate performance tables I wrote the following code:

import random
import datetime as dt
import pandas as pd
import logging
import platform, psutil, GPUtil
import csv

#for debugging purposes
logging.basicConfig(filename='stressTest_LOG.txt', level=logging.DEBUG, format=' %(asctime)s - %(levelname)s - %(message)s')
logging.disable(level=logging.DEBUG)

#enlarge pandas display area
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

def passGen(passLen, randSeed):
    # randSeed = None #None uses time stamp as a value
    # passLen = 15 #password length

    random.seed(a=randSeed, version=2)

    # populate lists with character ranges based of ASCII table
    letters = list(range(65, 91)) + list(range(97, 123))
    symbols = list(range(33, 48))
    numbers = list(range(48, 58))

    passCombined = letters + symbols + numbers
    random.shuffle(passCombined)

    # check if first element is from symbol list and if so replace with a number
    while passCombined[0] > 32 and passCombined[0] < 49:
        # print("First symbol: "+ str(chr(passCombined[0])))
        passCombined[0] = random.randint(48, 58)
        # print("Changed to: "+ str(chr(passCombined[0])))

    finalPassword = passCombined[slice(passLen)]

    return finalPassword


def showPass(password):
    if len(password) > 32:
        print("Invalid password length.\nHas to be less than 32 characters.")
        return -1

    print(''.join(str(chr(e)) for e in password))



####################################### Main #######################################

# Generate CSV file
with open('performanceResults2.csv', 'w', newline='') as f:

    #declare columns in CSV file and their order
    fieldnames = ['Action', 'Start Time', 'End Time', 'Time Elapsed', 'OS',
                  'System', 'RAM', 'CPU count', 'CPU freq', 'GPU']
    writer = csv.DictWriter(f, fieldnames=fieldnames)
    writer.writeheader()

    #gather system information
    info_architecture = platform.architecture()[0]
    info_machine = platform.machine()
    info_node = platform.node()
    info_system = platform.platform()
    info_os = platform.system()

    if info_os == 'Darwin':
        info_os = 'macOS'

    info_release = platform.release()
    info_version = platform.version()
    info_processor = platform.processor()
    info_pythonCompiler = platform.python_compiler()

    # get RAM memory info
    mem = psutil.virtual_memory().total
    mem = str(mem/(1024.**3)) + 'GB'

    # get CPU info
    cpu_count = psutil.cpu_count()
    cpu_freq = psutil.cpu_freq().current
    cpu_freq = round(cpu_freq / 1000, 2)
    cpu_freq = str(cpu_freq) + 'GHz'

    # get GPU info
    # Works only with Nvidia gpus and is based on nvidia-smi command
    gpuinfo = GPUtil.getGPUs()

    if len(gpuinfo) == 0:
        gpuinfo = 'Unsupported GPU model'

    #run random password generator program
    counter = 10000
    testCounter = 0


    #print("Test #1 Start time: " + str(startTime))


    for i in range(0,5):

        startTime = dt.datetime.now()

        while counter > 0:
            pass1 = passGen(30, None)
            #showPass(pass1)
            logging.debug('counter is: ' + str(counter) + ', password: ' + str(pass1))
            counter -= 1

        endTime = dt.datetime.now()
        #print("Test #1 End time  : " + str(endTime))

        timeDelta = endTime - startTime
        #print ("Test #1 Time elapsed: " + str(timeDelta))
        testCounter += 1
        counter = 10000
        testCounterDisplay = 'Test #' + str(testCounter)

        writer.writerow({'Action': testCounterDisplay, 'Start Time': startTime, 'End Time': endTime,
                         'Time Elapsed': timeDelta, 'OS': info_os, 'System': info_system, 'RAM': mem,
                         'CPU count': cpu_count, 'CPU freq': cpu_freq, 'GPU': gpuinfo})

#read back in and display the results
file = pd.read_csv('performanceResults2.csv', delimiter=',')
print(file)

And to compare the results I only got this far:

import pandas as pd
import numpy as np

#enlarge pandas display area
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

#read in data to compare
test1 = pd.read_csv('performanceResults1.csv', delimiter=',')
test2 = pd.read_csv('performanceResults2.csv', delimiter=',')

#check if dataframes are equal
equality = test1.equals(test2)
print('DataFrame equal: ', equality)


df1_filtered = pd.DataFrame(test1[['Time Elapsed']])
df2_filtered = pd.DataFrame(test2['Time Elapsed'])

Any thoughts?

Answer 1

Without seeing you time cell format it hard to help As I understand you time comes in datetime format in:

 dt.datetime.now()

if you want to convert to pandas timestamp:

 pd.to_datetime(dt.datetime.now())

you can run this on your column "Start Time" and "End Time" and reassign them. Check .dtypes() on your DataFrame, it may be "object" then run:

DF['Start Time'] = pd.to_datetime(DF['Start Time'])

after this dtype should be datetime64[ns] which will allow you to make calculations.