我正在try 从数据库获取数据,以便输入到机器学习模型.但我的生产者只将数据逐行放入数据库中,消费者也只能逐一获取数据.我的机器学习模型需要处理2000行.
我希望生产者和消费者每次都要放入和获得大量的行,而不仅仅是一个接一个
以下是我的python代码
def read_data():
torque_data_queryset = Torque.get_torque_data()
return list(torque_data_queryset)
def producer(queue):
print('Producer: Running')
data = read_data()
for item in data:
time.sleep(1)
queue.put(item)
print(f'> Producer added {item}')
queue.put(None)
print('Producer: Done')
# consumer task
def consumer(queue):
print('Consumer: Running')
while True:
item = queue.get()
print(item)
if item is None:
break
time.sleep(1)
print(f'> Consumer got {item}')
print('Consumer: Done')
def thread_function():
queue = Queue()
consumer_thread = Thread(target=consumer, args=(queue,))
consumer_thread.start()
producer_thread = Thread(target=producer, args=(queue,))
producer_thread.start()
producer_thread.join()
consumer_thread.join()
def thread_result(request):
thread_function()
return render(request, 'async_processing_result.html', {'message': 'Threads completed successfully!'})