Python源码示例：keras.utils.OrderedEnqueuer()
示例1
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例2
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例3
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例4
def test_ordered_enqueuer_fail_threads():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例5
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例6
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例7
def test_ordered_enqueuer_fail_processes():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例8
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例9
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例10
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例11
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例12
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例13
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例14
def test_ordered_enqueuer_fail_processes():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例15
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例16
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例17
def test_ordered_enqueuer_fail_threads():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例18
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例19
def test_context_switch():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer2 = OrderedEnqueuer(DummySequence([3, 200, 200, 3], value=15), use_multiprocessing=True)
enqueuer.start(3, 10)
enqueuer2.start(3, 10)
gen_output = enqueuer.get()
gen_output2 = enqueuer2.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[-1] == 99
# One epoch is completed so enqueuer will switch the Sequence
acc = []
for i in range(100):
acc.append(next(gen_output2)[0, 0, 0, 0])
assert acc[-1] == 99 * 15
# One epoch has been completed so enqueuer2 will switch
# Be sure that both Sequence were updated
assert next(gen_output)[0, 0, 0, 0] == 0
assert next(gen_output)[0, 0, 0, 0] == 5
assert next(gen_output2)[0, 0, 0, 0] == 0
assert next(gen_output2)[0, 0, 0, 0] == 15 * 5
# Tear down everything
enqueuer.stop()
enqueuer2.stop()
示例20
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例21
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例22
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例23
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例24
def test_ordered_enqueuer_fail_threads():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例25
def test_on_epoch_end_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(200):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc[100:] == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例26
def test_on_epoch_end_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list([k * 5 for k in range(100)]), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
示例27
def test_ordered_enqueuer_fail_processes():
enqueuer = OrderedEnqueuer(FaultSequence(), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
with pytest.raises(StopIteration):
next(gen_output)
示例28
def test_ordered_enqueuer_threads():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=False)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例29
def test_ordered_enqueuer_threads_not_ordered():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]),
use_multiprocessing=False,
shuffle=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc != list(range(100)), "Order was not keep in GeneratorEnqueuer with threads"
enqueuer.stop()
示例30
def test_ordered_enqueuer_processes():
enqueuer = OrderedEnqueuer(DummySequence([3, 200, 200, 3]), use_multiprocessing=True)
enqueuer.start(3, 10)
gen_output = enqueuer.get()
acc = []
for i in range(100):
acc.append(next(gen_output)[0, 0, 0, 0])
assert acc == list(range(100)), "Order was not keep in GeneratorEnqueuer with processes"
enqueuer.stop()
