Recently do AETA Earthquake prediction AI Algorithm contest This game , Look at him baseline Code , Found these lines ：
# Take the intersection of the three , Yes _magn file , Yes _sound file , And there are MagnUpdate & SoundUpdate
usable_stations = _continueable_stations & _set_magn & _set_sound
dump_object(Usable_Station_Path, usable_stations)
print(' Merge data :')
for type in ('magn', 'sound'):
res = []
for _id in tqdm(usable_stations, desc=f'{
type}:'):
# Data_Folder_Path+str(_id)+f'_{type}.csv' ==> e.g. : './data/19_magn.csv'
# Used_features Used to get the corresponding feature
_df = pd.read_csv(Data_Folder_Path+str(_id)+f'_{
type}.csv')[Used_features[type]]
res.append(_df)
final_df = pd.concat(res)
final_df.to_pickle(Merged_Data_Path[type]) # <-------------- Look here
del(final_df)
Why merge after reading , want .to_pickle Export to pkl What about the documents ？
Merge data :
magn:: 100%|██████████| 131/131 [11:44<00:00, 5.38s/it]
sound:: 100%|██████████| 131/131 [08:51<00:00, 4.05s/it]
Also note tqdm(usable_stations, desc=f'{type}:') This class , It comes with a :, No use desc Add :
A hundred times faster Pandas Performance optimization methods , Let your Pandas Fly up ！ in , To specify , Read csv、hdf and pkl In file , Read pkl Fastest format
His code has two more lines ：
# The earthquake in this area AETA Take out the station data
local_magn_data = magn_data[magn_data['StationID'].apply(lambda x:x in ID_list)].reset_index(drop=True)
wow , A batch of direct cards , Slow down ！！
intend ：
idx = magn_data['StationID'].apply(lambda x:x in ID_list) # find StationID That column , The elements are in ID_list The line of , If yes, here you are True, On the contrary, give False
local_magn_data = magn_data[idx] # Take those lines out
local_magn_data = local_magn_data.reset_index(drop=True) # Reset index
according to A hundred times faster Pandas Performance optimization methods , Let your Pandas Fly up ！ We try to use .isin Method to replace .apply(lambda x:x in ID_list)
local_magn_data = magn_data[magn_data['StationID'].isin(ID_list)].reset_index(drop=True)
Single step debugging after replacement 1s Just through ,Yes！
res_df[f'{
feature}_mean'] = None
res_df[f'{
feature}_max'] = None
res_df[f'{
feature}_min'] = None
res_df[f'{
feature}_max_min'] = None
for i,row in res_df.iterrows():
endDay = row['Day']
startDay = endDay - window
data_se = df[(df['Day']>startDay)&(df['Day']<=endDay)][feature]
res_df[f'{
feature}_mean'].iloc[i] = data_se.mean()
res_df[f'{
feature}_max'].iloc[i] = data_se.max()
res_df[f'{
feature}_min'].iloc[i] = data_se.min()
res_df[f'{
feature}_max_min'].iloc[i] = data_se.max() - data_se.min()
This is done by giving res_df[f'{feature}_mean'] Take a hole , But when we deal with it later , This column is object Of type
So the suggestion becomes
res_df[f'{
feature}_mean'] = 0.0
res_df[f'{
feature}_max'] = 0.0
res_df[f'{
feature}_min'] = 0.0
res_df[f'{
feature}_max_min'] = 0.0
These are the ways to speed up reading , Reminds me of before kaggle You can see it on [Reducing DataFrame memory size by ~65%] Reduce Pandas How to read memory ：
Just use the function he wrote ：
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
def reduce_mem_usage(props):
# Reference from ：
# https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65/notebook
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
NAlist = [] # Keeps track of columns that have missing values filled in.
for col in props.columns:
if props[col].dtype != object: # exclude strings
# Print... For the current column type
print("******************************")
print("Column: ",col)
print("dtype before: ",props[col].dtype)
# make variables for Int, max and min
IsInt = False
mx = props[col].max()
mn = props[col].min()
# Integer does not support NA, therefore, NA needs to be filled
# Integer I won't support it NA, So fill in
if not np.isfinite(props[col]).all():
NAlist.append(col)
props[col].fillna(mn-1,inplace=True)
# test if column can be converted to an integer
# Test whether the column can be converted to an integer
asint = props[col].fillna(0).astype(np.int64)
result = (props[col] - asint)
result = result.sum()
# If it's much worse , It can be transformed into int
if result > -0.01 and result < 0.01:
IsInt = True
# Make Integer/unsigned Integer datatypes
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
elif mx < 65535:
props[col] = props[col].astype(np.uint16)
elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
else:
props[col] = props[col].astype(np.uint64)
else:
if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:
props[col] = props[col].astype(np.int8)
elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:
props[col] = props[col].astype(np.int16)
elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:
props[col] = props[col].astype(np.int32)
elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:
props[col] = props[col].astype(np.int64)
# Make float datatypes 32 bit, Int If it cannot be transformed, it will be transformed into float32
else:
props[col] = props[col].astype(np.float32)
# Print new column type
print("dtype after: ",props[col].dtype)
print("******************************")
# Print final result
print("___MEMORY USAGE AFTER COMPLETION:___")
mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
return props, NAlist
Usage mode ：
props = pd.read_csv(r"../input/properties_2016.csv") #The properties dataset
# props To reduce the memory DataFrame,NAlist Is the name of the column with outliers
props, NAlist = reduce_mem_usage(props)
print("_________________")
print("")
print("Warning: the following columns have missing values filled with 'df['column_name'].min() -1': ")
print("_________________")
print("")
print(NAlist)
You can also refer to ：
How to Speed up Pandas by 4x with one line of code
How to Speed Up Pandas Calculations
About np.isfinite How to use it ：
>>> np.isfinite(1)
True
>>> np.isfinite(0)
True
>>> np.isfinite(np.nan)
False
>>> np.isfinite(np.inf)
False
>>> np.isfinite(np.NINF)
False
>>> np.isfinite([np.log(-1.),1.,np.log(0)])
array([False, True, False])
>>> x = np.array([-np.inf, 0., np.inf])
>>> y = np.array([2, 2, 2])
>>> np.isfinite(x, y)
array([0, 1, 0])
>>> y
array([0, 1, 0])
return True If x Not positive infinity 、 Negative infinity or NaN; Otherwise return to False. If x It's scalar , Then this is a scalar
