pkgs/development/python-modules/tensorflow/absl_to_std.patch

   1 diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor.cc y/tensorflow/c/experimental/stream_executor/stream_executor.cc
   2 --- x/tensorflow/c/experimental/stream_executor/stream_executor.cc      2023-10-19 14:25:48.648646416 +0000
   3 +++ y/tensorflow/c/experimental/stream_executor/stream_executor.cc      2023-10-19 14:29:11.700743574 +0000
   4 @@ -23,6 +23,7 @@
   5
   6  #include <string>
   7  #include <utility>
   8 +#include <optional>
   9
  10  #include "absl/functional/any_invocable.h"
  11  #include "tensorflow/c/c_api_macros.h"
  12 @@ -275,17 +276,17 @@
  13      stream_executor_->unified_memory_deallocate(&device_, mem);
  14    }
  15
  16 -  absl::optional<AllocatorStats> GetAllocatorStats() override {
  17 +  std::optional<AllocatorStats> GetAllocatorStats() override {
  18      SP_AllocatorStats c_stats{SP_ALLOCATORSTATS_STRUCT_SIZE};
  19      TF_Bool has_stats =
  20          stream_executor_->get_allocator_stats(&device_, &c_stats);
  21      if (!has_stats) {
  22 -      return absl::nullopt;
  23 +      return std::nullopt;
  24      }
  25      tsl::Status status = ValidateSPAllocatorStats(c_stats);
  26      if (!status.ok()) {
  27        LOG(ERROR) << status.message();
  28 -      return absl::nullopt;
  29 +      return std::nullopt;
  30      }
  31      ::stream_executor::AllocatorStats stats;
  32      stats.num_allocs = c_stats.num_allocs;
  33 diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc
  34 --- x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:25:48.648646416 +0000
  35 +++ y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc 2023-10-19 14:29:11.700743574 +0000
  36 @@ -15,6 +15,7 @@
  37  #include "tensorflow/c/experimental/stream_executor/stream_executor.h"
  38
  39  #include <utility>
  40 +#include <optional>
  41
  42  #include "tensorflow/c/experimental/stream_executor/stream_executor_internal.h"
  43  #include "tensorflow/c/experimental/stream_executor/stream_executor_test_util.h"
  44 @@ -239,7 +240,7 @@
  45    };
  46
  47    StreamExecutor* executor = GetExecutor(0);
  48 -  absl::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats();
  49 +  std::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats();
  50    ASSERT_TRUE(optional_stats.has_value());
  51    AllocatorStats stats = optional_stats.value();
  52    ASSERT_EQ(stats.bytes_in_use, 123);
  53 diff -Naurd x/tensorflow/compiler/xla/stream_executor/allocator_stats.h y/tensorflow/compiler/xla/stream_executor/allocator_stats.h
  54 --- x/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:25:55.064649379 +0000
  55 +++ y/tensorflow/compiler/xla/stream_executor/allocator_stats.h 2023-10-19 14:29:11.700743574 +0000
  56 @@ -17,6 +17,7 @@
  57  #define TENSORFLOW_COMPILER_XLA_STREAM_EXECUTOR_ALLOCATOR_STATS_H_
  58
  59  #include <string>
  60 +#include <optional>
  61
  62  #include "absl/types/optional.h"
  63  #include "tensorflow/compiler/xla/stream_executor/platform/port.h"
  64 diff -Naurd x/tensorflow/core/common_runtime/mkl_cpu_allocator.h y/tensorflow/core/common_runtime/mkl_cpu_allocator.h
  65 --- x/tensorflow/core/common_runtime/mkl_cpu_allocator.h        2023-10-19 14:25:58.996651199 +0000
  66 +++ y/tensorflow/core/common_runtime/mkl_cpu_allocator.h        2023-10-19 14:29:11.700743574 +0000
  67 @@ -22,6 +22,7 @@
  68  #ifdef INTEL_MKL
  69
  70  #include <cstdlib>
  71 +#include <optional>
  72
  73  #include "tensorflow/core/common_runtime/bfc_allocator.h"
  74  #include "tensorflow/core/common_runtime/pool_allocator.h"
  75 @@ -80,7 +81,7 @@
  76      port::AlignedFree(ptr);
  77    }
  78
  79 -  absl::optional<AllocatorStats> GetStats() override {
  80 +  std::optional<AllocatorStats> GetStats() override {
  81      mutex_lock l(mutex_);
  82      return stats_;
  83    }
  84 @@ -242,7 +243,7 @@
  85        large_size_allocator_->DeallocateRaw(ptr);
  86      }
  87    }
  88 -  absl::optional<AllocatorStats> GetStats() override {
  89 +  std::optional<AllocatorStats> GetStats() override {
  90      auto s_stats = small_size_allocator_->GetStats();
  91      auto l_stats = large_size_allocator_->GetStats();
  92
  93 diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc
  94 --- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc      2023-10-19 14:25:59.236651310 +0000
  95 +++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc      2023-10-19 14:29:11.700743574 +0000
  96 @@ -32,7 +32,7 @@
  97    return sub_allocator_->Free(ptr, 0);
  98  }
  99
 100 -absl::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() {
 101 +std::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() {
 102    AllocatorStats stats_;
 103    stats_.num_allocs = 0;
 104    stats_.peak_bytes_in_use = 0;
 105 diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h
 106 --- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h       2023-10-19 14:25:59.236651310 +0000
 107 +++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h       2023-10-19 14:29:11.700743574 +0000
 108 @@ -19,6 +19,7 @@
 109  #include <string>
 110  #include <unordered_map>
 111  #include <vector>
 112 +#include <optional>
 113
 114  #include "tensorflow/core/common_runtime/device/device_mem_allocator.h"
 115  #include "tensorflow/core/platform/thread_annotations.h"
 116 @@ -37,7 +38,7 @@
 117
 118    bool TracksAllocationSizes() const override { return false; }
 119    string Name() override { return "Simple allocator"; }
 120 -  absl::optional<AllocatorStats> GetStats() override;
 121 +  std::optional<AllocatorStats> GetStats() override;
 122
 123    AllocatorMemoryType GetMemoryType() const override {
 124      return sub_allocator_->GetMemoryType();
 125 diff -Naurd x/tensorflow/core/common_runtime/process_state.h y/tensorflow/core/common_runtime/process_state.h
 126 --- x/tensorflow/core/common_runtime/process_state.h    2023-10-19 14:25:59.076651236 +0000
 127 +++ y/tensorflow/core/common_runtime/process_state.h    2023-10-19 14:29:11.704743576 +0000
 128 @@ -20,6 +20,7 @@
 129  #include <map>
 130  #include <unordered_map>
 131  #include <vector>
 132 +#include <optional>
 133
 134  #include "tensorflow/core/framework/allocator.h"
 135  #include "tensorflow/core/framework/allocator_registry.h"
 136 @@ -144,7 +145,7 @@
 137    size_t AllocatedSize(const void* p) const override {
 138      return a_->AllocatedSize(p);
 139    }
 140 -  absl::optional<AllocatorStats> GetStats() override { return a_->GetStats(); }
 141 +  std::optional<AllocatorStats> GetStats() override { return a_->GetStats(); }
 142    bool ClearStats() override { return a_->ClearStats(); }
 143
 144    AllocatorMemoryType GetMemoryType() const override {
 145 diff -Naurd x/tensorflow/core/common_runtime/step_stats_collector.cc y/tensorflow/core/common_runtime/step_stats_collector.cc
 146 --- x/tensorflow/core/common_runtime/step_stats_collector.cc    2023-10-19 14:25:59.112651253 +0000
 147 +++ y/tensorflow/core/common_runtime/step_stats_collector.cc    2023-10-19 14:29:11.704743576 +0000
 148 @@ -15,6 +15,7 @@
 149  #include "tensorflow/core/common_runtime/step_stats_collector.h"
 150
 151  #include <memory>
 152 +#include <optional>
 153
 154  #include "tensorflow/core/common_runtime/costmodel_manager.h"
 155  #include "tensorflow/core/framework/allocation_description.pb.h"
 156 @@ -175,7 +176,7 @@
 157    memory->set_peak_bytes(std::get<1>(sizes));
 158    memory->set_live_bytes(std::get<2>(sizes));
 159
 160 -  absl::optional<AllocatorStats> stats = allocator->GetStats();
 161 +  std::optional<AllocatorStats> stats = allocator->GetStats();
 162    if (stats) {
 163      memory->set_allocator_bytes_in_use(stats->bytes_in_use);
 164    }
 165 diff -Naurd x/tensorflow/core/framework/allocator_test.cc y/tensorflow/core/framework/allocator_test.cc
 166 --- x/tensorflow/core/framework/allocator_test.cc       2023-10-19 14:25:59.524651443 +0000
 167 +++ y/tensorflow/core/framework/allocator_test.cc       2023-10-19 14:29:11.704743576 +0000
 168 @@ -17,6 +17,7 @@
 169
 170  #include <algorithm>
 171  #include <vector>
 172 +#include <optional>
 173
 174  #include "tensorflow/core/framework/typed_allocator.h"
 175  #include "tensorflow/core/platform/logging.h"
 176 @@ -33,7 +34,7 @@
 177
 178  static void CheckStats(Allocator* a, int64_t num_allocs, int64_t bytes_in_use,
 179                         int64_t peak_bytes_in_use, int64_t largest_alloc_size) {
 180 -  absl::optional<AllocatorStats> stats = a->GetStats();
 181 +  std::optional<AllocatorStats> stats = a->GetStats();
 182    EXPECT_TRUE(stats);
 183    if (!stats) {
 184      return;
 185 @@ -255,7 +256,7 @@
 186    EXPECT_EQ(e0.Name(), "MemoryAllocation")
 187        << "XSpace: " << xspace.DebugString();
 188    {
 189 -    absl::optional<std::string> bytes_allocated, peak_bytes_in_use,
 190 +    std::optional<std::string> bytes_allocated, peak_bytes_in_use,
 191          requested_bytes, allocation_bytes;
 192      e0.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) {
 193        LOG(ERROR) << "STAT " << stat.Name() << ": " << stat.ToString();
 194 @@ -282,7 +283,7 @@
 195    EXPECT_EQ(e1.Name(), "MemoryDeallocation")
 196        << "XSpace: " << xspace.DebugString();
 197    {
 198 -    absl::optional<std::string> bytes_allocated, peak_bytes_in_use,
 199 +    std::optional<std::string> bytes_allocated, peak_bytes_in_use,
 200          allocation_bytes;
 201      e1.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) {
 202        if (stat.Name() == "bytes_allocated") {
 203 diff -Naurd x/tensorflow/core/framework/tracking_allocator_test.cc y/tensorflow/core/framework/tracking_allocator_test.cc
 204 --- x/tensorflow/core/framework/tracking_allocator_test.cc      2023-10-19 14:25:59.700651525 +0000
 205 +++ y/tensorflow/core/framework/tracking_allocator_test.cc      2023-10-19 14:29:11.704743576 +0000
 206 @@ -16,6 +16,7 @@
 207  #include "tensorflow/core/framework/tracking_allocator.h"
 208
 209  #include <unordered_map>
 210 +#include <optional>
 211
 212  #include "tensorflow/core/framework/allocator.h"
 213  #include "tensorflow/core/platform/logging.h"
 214 @@ -44,7 +45,7 @@
 215      EXPECT_NE(size_map_.end(), iter);
 216      return iter->second;
 217    }
 218 -  absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; }
 219 +  std::optional<AllocatorStats> GetStats() override { return std::nullopt; }
 220
 221   private:
 222    std::unordered_map<const void*, size_t> size_map_;
 223 @@ -58,7 +59,7 @@
 224    }
 225    void DeallocateRaw(void* ptr) override {}
 226    bool TracksAllocationSizes() const override { return true; }
 227 -  absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; }
 228 +  std::optional<AllocatorStats> GetStats() override { return std::nullopt; }
 229  };
 230
 231  TEST(TrackingAllocatorTest, SimpleNoTracking) {
 232 diff -Naurd x/tensorflow/core/grappler/clusters/single_machine.cc y/tensorflow/core/grappler/clusters/single_machine.cc
 233 --- x/tensorflow/core/grappler/clusters/single_machine.cc       2023-10-19 14:25:59.964651648 +0000
 234 +++ y/tensorflow/core/grappler/clusters/single_machine.cc       2023-10-19 14:29:11.704743576 +0000
 235 @@ -17,6 +17,7 @@
 236
 237  #include <atomic>
 238  #include <memory>
 239 +#include <optional>
 240
 241  #include "tensorflow/cc/training/queue_runner.h"
 242  #include "tensorflow/core/common_runtime/device.h"
 243 @@ -230,7 +231,7 @@
 244        return Status(absl::StatusCode::kInvalidArgument,
 245                      "Tracking allocation is not enabled.");
 246      }
 247 -    absl::optional<AllocatorStats> stats = allocator->GetStats();
 248 +    std::optional<AllocatorStats> stats = allocator->GetStats();
 249      (*device_peak_memory)[device->name()] =
 250          (stats ? stats->peak_bytes_in_use : 0);
 251    }
 252 diff -Naurd x/tensorflow/core/kernels/stack.cc y/tensorflow/core/kernels/stack.cc
 253 --- x/tensorflow/core/kernels/stack.cc  2023-10-19 14:26:01.668652437 +0000
 254 +++ y/tensorflow/core/kernels/stack.cc  2023-10-19 14:29:11.704743576 +0000
 255 @@ -18,6 +18,7 @@
 256  #include <limits.h>
 257  #include <atomic>
 258  #include <vector>
 259 +#include <optional>
 260
 261  #include "tensorflow/core/common_runtime/device.h"
 262  #include "tensorflow/core/framework/device_base.h"
 263 @@ -245,7 +246,7 @@
 264      DeviceContext* device_ctxt = ctx->op_device_context();
 265      auto device = static_cast<tensorflow::Device*>(ctx->device());
 266      Allocator* allocator = device->GetAllocator(alloc_attrs);
 267 -    absl::optional<AllocatorStats> stats = allocator->GetStats();
 268 +    std::optional<AllocatorStats> stats = allocator->GetStats();
 269      if (stats && *stats->bytes_limit &&
 270          stats->bytes_in_use > (*stats->bytes_limit * kOccupancy)) {
 271        // Asynchronously copy the tensor from GPU to CPU memory.
 272 diff -Naurd x/tensorflow/python/tfe_wrapper.cc y/tensorflow/python/tfe_wrapper.cc
 273 --- x/tensorflow/python/tfe_wrapper.cc  2023-10-19 14:26:10.716656639 +0000
 274 +++ y/tensorflow/python/tfe_wrapper.cc  2023-10-19 14:29:11.708743578 +0000
 275 @@ -14,6 +14,7 @@
 276  ==============================================================================*/
 277
 278  #include <memory>
 279 +#include <optional>
 280
 281  #include "Python.h"
 282  #include "absl/strings/match.h"
 283 @@ -691,7 +692,7 @@
 284      tensorflow::AllocatorAttributes attrs;
 285      tensorflow::Allocator* allocator = matched_device->GetAllocator(attrs);
 286
 287 -    if (absl::optional<tensorflow::AllocatorStats> stats =
 288 +    if (std::optional<tensorflow::AllocatorStats> stats =
 289              allocator->GetStats()) {
 290        return std::map<std::string, int64_t>{{"current", stats->bytes_in_use},
 291                                              {"peak", stats->peak_bytes_in_use}};
 292 diff -Naurd x/tensorflow/tsl/framework/allocator.h y/tensorflow/tsl/framework/allocator.h
 293 --- x/tensorflow/tsl/framework/allocator.h      2023-10-19 14:26:15.884659044 +0000
 294 +++ y/tensorflow/tsl/framework/allocator.h      2023-10-19 14:29:11.708743578 +0000
 295 @@ -216,7 +216,7 @@
 296    }
 297
 298    // Fills in 'stats' with statistics collected by this allocator.
 299 -  virtual absl::optional<AllocatorStats> GetStats() { return absl::nullopt; }
 300 +  virtual std::optional<AllocatorStats> GetStats() { return std::nullopt; }
 301
 302    // If implemented, clears the internal stats except for the `in_use` fields
 303    // and sets the `peak_bytes_in_use` to be equal to the `bytes_in_use`. Returns
 304 diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.cc y/tensorflow/tsl/framework/bfc_allocator.cc
 305 --- x/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:26:15.900659052 +0000
 306 +++ y/tensorflow/tsl/framework/bfc_allocator.cc 2023-10-19 14:29:11.708743578 +0000
 307 @@ -1205,7 +1205,7 @@
 308    return md;
 309  }
 310
 311 -absl::optional<AllocatorStats> BFCAllocator::GetStats() {
 312 +std::optional<AllocatorStats> BFCAllocator::GetStats() {
 313    mutex_lock l(lock_);
 314    return stats_;
 315  }
 316 diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.h y/tensorflow/tsl/framework/bfc_allocator.h
 317 --- x/tensorflow/tsl/framework/bfc_allocator.h  2023-10-19 14:26:15.900659052 +0000
 318 +++ y/tensorflow/tsl/framework/bfc_allocator.h  2023-10-19 14:29:11.708743578 +0000
 319 @@ -22,6 +22,7 @@
 320  #include <string>
 321  #include <unordered_map>
 322  #include <vector>
 323 +#include <optional>
 324
 325  #include "absl/container/flat_hash_set.h"
 326  #include "tensorflow/tsl/framework/allocator.h"
 327 @@ -93,7 +94,7 @@
 328
 329    int64_t AllocationId(const void* ptr) const override;
 330
 331 -  absl::optional<AllocatorStats> GetStats() override;
 332 +  std::optional<AllocatorStats> GetStats() override;
 333
 334    bool ClearStats() override;
 335
 336 diff -Naurd x/tensorflow/tsl/framework/cpu_allocator_impl.cc y/tensorflow/tsl/framework/cpu_allocator_impl.cc
 337 --- x/tensorflow/tsl/framework/cpu_allocator_impl.cc    2023-10-19 14:26:15.928659065 +0000
 338 +++ y/tensorflow/tsl/framework/cpu_allocator_impl.cc    2023-10-19 14:29:11.708743578 +0000
 339 @@ -15,6 +15,7 @@
 340
 341  #include <algorithm>
 342  #include <atomic>
 343 +#include <optional>
 344
 345  #include "tensorflow/tsl/framework/allocator.h"
 346  #include "tensorflow/tsl/framework/allocator_registry.h"
 347 @@ -145,8 +146,8 @@
 348          /*level=*/tsl::profiler::TraceMeLevel::kInfo);
 349    }
 350
 351 -  absl::optional<AllocatorStats> GetStats() override {
 352 -    if (!cpu_allocator_collect_stats) return absl::nullopt;
 353 +  std::optional<AllocatorStats> GetStats() override {
 354 +    if (!cpu_allocator_collect_stats) return std::nullopt;
 355      mutex_lock l(mu_);
 356      return stats_;
 357    }
 358 diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.cc y/tensorflow/tsl/framework/tracking_allocator.cc
 359 --- x/tensorflow/tsl/framework/tracking_allocator.cc    2023-10-19 14:26:15.968659084 +0000
 360 +++ y/tensorflow/tsl/framework/tracking_allocator.cc    2023-10-19 14:29:11.708743578 +0000
 361 @@ -152,7 +152,7 @@
 362    }
 363  }
 364
 365 -absl::optional<AllocatorStats> TrackingAllocator::GetStats() {
 366 +std::optional<AllocatorStats> TrackingAllocator::GetStats() {
 367    return allocator_->GetStats();
 368  }
 369
 370 diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.h y/tensorflow/tsl/framework/tracking_allocator.h
 371 --- x/tensorflow/tsl/framework/tracking_allocator.h     2023-10-19 14:26:15.968659084 +0000
 372 +++ y/tensorflow/tsl/framework/tracking_allocator.h     2023-10-19 14:29:11.712743580 +0000
 373 @@ -17,6 +17,7 @@
 374  #define TENSORFLOW_TSL_FRAMEWORK_TRACKING_ALLOCATOR_H_
 375
 376  #include <unordered_map>
 377 +#include <optional>
 378
 379  #include "tensorflow/tsl/framework/allocator.h"
 380  #include "tensorflow/tsl/lib/gtl/inlined_vector.h"
 381 @@ -66,7 +67,7 @@
 382    size_t RequestedSize(const void* ptr) const override;
 383    size_t AllocatedSize(const void* ptr) const override;
 384    int64_t AllocationId(const void* ptr) const override;
 385 -  absl::optional<AllocatorStats> GetStats() override;
 386 +  std::optional<AllocatorStats> GetStats() override;
 387    bool ClearStats() override;
 388
 389    AllocatorMemoryType GetMemoryType() const override {