在Python中处理大量CSV文件中的数据

将所有CSV文件连接到一个收件箱中，groupby:

import os
import pandas as pd
csv_folder = r"/home/bera/Desktop/gistest/csvs/" #The folder with the 102 csv files, and nothing else.

#Find and create a dataframe of each csv file
frame_list = []
for root, folder, files in os.walk(csv_folder):
    for file in files:
        if file.endswith("csv"):
            fullname = os.path.join(root, file)
            df = pd.read_csv(fullname, sep=";")
            df["filename"] = file #Store the filename as a value
            frame_list.append(df) 
print(len(frame_list))
#102

megaframe = pd.concat(frame_list, axis=0) #Concatenate the 102 frames into one
print(megaframe.shape)
#(1020000, 3)

print(megaframe.sample(3))
#                  w         s     filename
# 3940    884.419539  0.018198  file_93.csv
# 6314   1526.922029  0.013237   file_4.csv
# 3322  18177.653004  0.003916  file_51.csv

megaframe["s_sq"] = megaframe["s"]**2
df_means = megaframe.groupby("filename", as_index=False)["s_sq"].mean()
print(df_means.head())
#        filename      s_sq
# 0    file_0.csv  0.000134
# 1    file_1.csv  0.000135
# 2   file_10.csv  0.000132
# 3  file_100.csv  0.000135
# 4  file_101.csv  0.000132