45 lines
1.4 KiB
Python
45 lines
1.4 KiB
Python
from datasets import load_dataset
|
|
import requests
|
|
from PIL import Image
|
|
from io import BytesIO
|
|
import pandas as pd
|
|
|
|
ds = load_dataset("ckandemir/amazon-products")
|
|
valid_data = []
|
|
processed_count = 0 # 用于追踪成功处理的数量
|
|
|
|
# 遍历数据集进行验证
|
|
for idx, item in enumerate(ds['train']):
|
|
# 检查Product Name是否为空
|
|
if not item['Product Name'] or pd.isna(item['Product Name']):
|
|
continue
|
|
|
|
# 验证图片URL是否可以加载
|
|
try:
|
|
response = requests.get(item['Image'], timeout=5)
|
|
if response.status_code == 200:
|
|
# 尝试将图片加载为PIL Image对象
|
|
image = Image.open(BytesIO(response.content))
|
|
# 如果成功加载，保存图片对象和产品名称
|
|
valid_data.append({
|
|
'image': image,
|
|
'name': item['Product Name']
|
|
})
|
|
|
|
processed_count += 1
|
|
# 每成功处理100条数据输出一次信息
|
|
if processed_count % 100 == 0:
|
|
print(f"已成功处理 {processed_count} 条数据，当前处理到第 {idx+1} 条")
|
|
except Exception as e:
|
|
continue
|
|
|
|
print(f"\n处理完成！")
|
|
print(f"原始数据数量: {len(ds['train'])}")
|
|
print(f"清洗后的有效数据数量: {len(valid_data)}")
|
|
|
|
# 保存处理后的数据
|
|
import pickle
|
|
with open('valid_products.pkl', 'wb') as f:
|
|
pickle.dump(valid_data, f)
|
|
|