Arrow 之 Buffer

Buffer 转成数据类型

std::vector<std::shared_ptr<arrow::Buffer>> buffers;
  std::vector<int64_t> in_buf_addrs;
  std::vector<int64_t> in_buf_sizes;
  for (int i = 0; i < rb_schema->num_fields(); ++i) {
    ASSERT_NOT_OK(AppendBuffers(input_batch_arr->column(i), &buffers));
  }
  std::cout <<  "buffers.size() " << buffers.size() << std::endl;
  auto bufferIndex = 0;
  for (auto buffer : buffers) {
    std::cout <<  "bufferIndex: " << bufferIndex << std::endl;
    if (buffer == nullptr) {
      std::cout <<  "buffer == nullptr " << std::endl;
      continue;
    }
    std::cout <<  "buffer->ToHexString(): " << buffer->ToHexString() << std::endl;
    std::cout <<  "buffer->ToString(): " << buffer->ToString() << std::endl;

    uint8_t* buffer_data = buffer->mutable_data();
    int32_t val = *(int32_t*)buffer_data;
    int32_t val2 = *((int32_t*)buffer_data+1);
    std::cout << "buffer val: " << val << std::endl;
    std::cout << "buffer val2: " << val2 << std::endl;
    bufferIndex++;
  }

示例

uint8_t* address = columnarToRowConverter->GetBufferAddress();
  std::cout << "uint8_t: \n";
  for(int i=0; i< 16; i++) {
    std::cout << "*(address+" << i << ")" << *(address+i) << std::endl;
  }

  std::cout << "uint16_t: \n";
  for(int i=0; i< 16; i++) {
    std::cout << "*(address+" << i << ")" << (uint16_t)*(address+i) << std::endl;
  }

uint8_t:
*(address+0)
*(address+1)
*(address+2)
*(address+3)
*(address+4)
*(address+5)
*(address+6)
*(address+7)
*(address+8)
*(address+9)
*(address+10)
*(address+11)
*(address+12)
*(address+13)
*(address+14)
*(address+15)
uint16_t:
*(address+0)0
*(address+1)0
*(address+2)0
*(address+3)0
*(address+4)0
*(address+5)0
*(address+6)0
*(address+7)0
*(address+8)1
*(address+9)0
*(address+10)0
*(address+11)0
*(address+12)0
*(address+13)0
*(address+14)0
*(address+15)0

Buffer 接口

cpp/src/arrow/buffer.cc

Resize(size, false) 近似 等于 Reserve（size）

  Status Resize(const int64_t new_size, bool shrink_to_fit = true) override {
    if (ARROW_PREDICT_FALSE(new_size < 0)) {
      return Status::Invalid("Negative buffer resize: ", new_size);
    }
    if (mutable_data_ && shrink_to_fit && new_size <= size_) {
      // Buffer is non-null and is not growing, so shrink to the requested size without
      // excess space.
      int64_t new_capacity = BitUtil::RoundUpToMultipleOf64(new_size);
      if (capacity_ != new_capacity) {
        // Buffer hasn't got yet the requested size.
        RETURN_NOT_OK(pool_->Reallocate(capacity_, new_capacity, &mutable_data_));
        data_ = mutable_data_;
        capacity_ = new_capacity;
      }
    } else {
      RETURN_NOT_OK(Reserve(new_size));
    }
    size_ = new_size;

    return Status::OK();
  }

 Status Reserve(const int64_t capacity) override {
    if (capacity < 0) {
      return Status::Invalid("Negative buffer capacity: ", capacity);
    }
    if (!mutable_data_ || capacity > capacity_) {
      uint8_t* new_data;
      int64_t new_capacity = BitUtil::RoundUpToMultipleOf64(capacity);
      if (mutable_data_) {
        RETURN_NOT_OK(pool_->Reallocate(capacity_, new_capacity, &mutable_data_));
      } else {
        RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
        mutable_data_ = new_data;
      }
      data_ = mutable_data_;
      capacity_ = new_capacity;
    }
    return Status::OK();
  }