Skip to content

Commit

Permalink
Fix
Browse files Browse the repository at this point in the history
  • Loading branch information
co63oc committed Oct 22, 2024
1 parent 9d5cd38 commit 0454ef9
Showing 1 changed file with 1 addition and 67 deletions.
68 changes: 1 addition & 67 deletions paddle/fluid/operators/reader/create_py_reader_op.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,73 +26,7 @@ class CreatePyReaderOp : public framework::OperatorBase {

private:
void RunImpl(const framework::Scope& scope,
const phi::Place& dev_place) const override {
auto* out = scope.FindVar(Output("Out"))
->template GetMutable<framework::ReaderHolder>();
if (out->Get() != nullptr) return;

const std::string& queue_name = Input("blocking_queue");
auto* queue_holder_var = scope.FindVar(queue_name);
PADDLE_ENFORCE_NOT_NULL(
queue_holder_var,
common::errors::NotFound(
"No LoDTensorBlockingQueueHolder variable with name %s found. This "
"may be because the DataLoader is defined in another Scope, "
"which is different from the Scope when calling Executor.run.",
queue_name));
std::shared_ptr<LoDTensorBlockingQueue> queue;
std::shared_ptr<OrderedMultiDeviceLoDTensorBlockingQueue> ordered_queue;
int dev_idx = -1;
if (queue_holder_var->IsType<LoDTensorBlockingQueueHolder>()) {
queue = queue_holder_var->Get<LoDTensorBlockingQueueHolder>().GetQueue();
} else if (queue_holder_var
->IsType<OrderedMultiDeviceLoDTensorBlockingQueueHolder>()) {
auto* queue_holder =
queue_holder_var
->GetMutable<OrderedMultiDeviceLoDTensorBlockingQueueHolder>();
dev_idx = Attr<int>("device_index");
ordered_queue = queue_holder->GetQueue();
ordered_queue->SetDeviceCount(Attr<int>("device_count"));
queue = ordered_queue->GetQueue(dev_idx);
}

/* Converting shape_concat and ranks into DDim of each data.
shape_concat and ranks are shapes and shape ranks of each data.E.g.
shape_concat = [2,3,4,5,6], ranks = [3,2] means two data whose shapes are
[2,3,4] and [5,6] respectively. */
auto& shape_concat = Attr<std::vector<int>>("shape_concat");
auto& ranks = Attr<std::vector<int>>("ranks");
int shape_start_index = 0;
std::vector<phi::DDim> dims;
for (auto rank : ranks) {
int shape_end_index = shape_start_index + rank;
auto shape = std::vector<int>(shape_concat.begin() + shape_start_index,
shape_concat.begin() + shape_end_index);
dims.push_back(common::make_ddim(shape));
shape_start_index = shape_end_index;
}

// Converts VarType from int to enum
auto& dtype_int = Attr<std::vector<int>>("dtypes");
std::vector<framework::proto::VarType::Type> var_types;
for (auto type_int : dtype_int) {
var_types.push_back(
static_cast<framework::proto::VarType::Type>(type_int));
}

// Converts need_check_feed from int to bool
auto& need_check_feed_int = Attr<std::vector<int>>("need_check_feed");
std::vector<bool> need_check_feed;
for (auto feed_int : need_check_feed_int) {
need_check_feed.push_back(static_cast<bool>(feed_int));
}
auto py_reader =
std::make_shared<PyReader>(queue, dims, var_types, need_check_feed);
if (ordered_queue) {
ordered_queue->SetResetMethod(dev_idx, [out] { out->Clear(); });
}
out->Reset(py_reader);
}
const phi::Place& dev_place) const override {}
};

class CreatePyReaderOpMaker : public FileReaderMakerBase {
Expand Down

0 comments on commit 0454ef9

Please sign in to comment.