YAKL_FArray.h

Go to the documentation of this file.

 
#pragma once
// Included by YAKL_Array.h
 
__YAKL_NAMESPACE_WRAPPER_BEGIN__
namespace yakl {
 
  template <class T, int rank, int myMem>
  class Array<T,rank,myMem,styleFortran> : public ArrayBase<T,rank,myMem,styleFortran> {
  public:
 
    typedef typename std::remove_cv<T>::type       type;
    typedef          T                             value_type;
    typedef typename std::add_const<type>::type    const_value_type;
    typedef typename std::remove_const<type>::type non_const_value_type;
 
    //* @private */
    int lbounds[rank];  // Lower bounds
 
 
    // Start off all constructors making sure the pointers are null
    YAKL_INLINE void nullify() {
      this->myData   = nullptr;
      this->refCount = nullptr;
      for (int i=0; i < rank; i++) { this->lbounds[i] = 1; this->dimension[i] = 0; }
      #ifdef YAKL_DEBUG
        this->myname="Uninitialized";
      #endif
    }
 
    /* CONSTRUCTORS
    Always nullify before beginning so that myData == nullptr upon init.
    */
    YAKL_INLINE Array() {
      nullify();
    }
    YAKL_INLINE explicit Array(char const * label) {
      nullify();
      #ifdef YAKL_DEBUG
        this->myname = label;
      #endif
    }
 
 
    YAKL_INLINE Array( char const* label , Bnd b1 ) : Array(label,Bnds(b1)) {
      static_assert( rank == 1 , "ERROR: Calling constructor with 1 bound on non-rank-1 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ) : Array(label,Bnds(b1,b2)) {
      static_assert( rank == 2 , "ERROR: Calling constructor with 2 bound on non-rank-2 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ) : Array(label,Bnds(b1,b2,b3)) {
      static_assert( rank == 3 , "ERROR: Calling constructor with 3 bound on non-rank-3 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ,
                                           Bnd b4 ) : Array(label,Bnds(b1,b2,b3,b4)) {
      static_assert( rank == 4 , "ERROR: Calling constructor with 4 bound on non-rank-4 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ,
                                           Bnd b4 ,
                                           Bnd b5 ) : Array(label,Bnds(b1,b2,b3,b4,b5)) {
      static_assert( rank == 5 , "ERROR: Calling constructor with 5 bound on non-rank-5 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ,
                                           Bnd b4 ,
                                           Bnd b5 ,
                                           Bnd b6 ) : Array(label,Bnds(b1,b2,b3,b4,b5,b6)) {
      static_assert( rank == 6 , "ERROR: Calling constructor with 6 bound on non-rank-6 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ,
                                           Bnd b4 ,
                                           Bnd b5 ,
                                           Bnd b6 ,
                                           Bnd b7 ) : Array(label,Bnds(b1,b2,b3,b4,b5,b6,b7)) {
      static_assert( rank == 7 , "ERROR: Calling constructor with 7 bound on non-rank-7 array" );
    }
    YAKL_INLINE Array( char const* label , Bnd b1 ,
                                           Bnd b2 ,
                                           Bnd b3 ,
                                           Bnd b4 ,
                                           Bnd b5 ,
                                           Bnd b6 ,
                                           Bnd b7 ,
                                           Bnd b8 ) : Array(label,Bnds(b1,b2,b3,b4,b5,b6,b7,b8)) {
      static_assert( rank == 8 , "ERROR: Calling constructor with 8 bound on non-rank-8 array" );
    }
    YAKL_INLINE Array(char const * label, Bnds bnds) {
      static_assert( rank >= 1 && rank <= 8 , "ERROR: Creating Array with a rank < 1 or > 8" );
      nullify();
      #ifdef YAKL_DEBUG
        if ( bnds.size() < rank ) { yakl_throw("ERROR: Number of array bounds specified is < rank"); }
      #endif
      YAKL_EXECUTE_ON_HOST_ONLY( this->deallocate(); )
      #ifdef YAKL_DEBUG
        this->myname = label;
      #endif
      for (int i=0; i < rank; i++) { this->lbounds[i] = bnds[i].l; this->dimension[i] = bnds[i].u - bnds[i].l + 1; }
      YAKL_EXECUTE_ON_HOST_ONLY( this->allocate(); )
    }
 
 
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ) : Array(label,data,Bnds(b1)) {
      static_assert( rank == 1 , "ERROR: Calling constructor with 1 bound on non-rank-1 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ) : Array(label,data,Bnds(b1,b2)) {
      static_assert( rank == 2 , "ERROR: Calling constructor with 2 bound on non-rank-2 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ) : Array(label,data,Bnds(b1,b2,b3)) {
      static_assert( rank == 3 , "ERROR: Calling constructor with 3 bound on non-rank-3 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ,
                                                    Bnd b4 ) : Array(label,data,Bnds(b1,b2,b3,b4)) {
      static_assert( rank == 4 , "ERROR: Calling constructor with 4 bound on non-rank-4 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ,
                                                    Bnd b4 ,
                                                    Bnd b5 ) : Array(label,data,Bnds(b1,b2,b3,b4,b5)) {
      static_assert( rank == 5 , "ERROR: Calling constructor with 5 bound on non-rank-5 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ,
                                                    Bnd b4 ,
                                                    Bnd b5 ,
                                                    Bnd b6 ) : Array(label,data,Bnds(b1,b2,b3,b4,b5,b6)) {
      static_assert( rank == 6 , "ERROR: Calling constructor with 6 bound on non-rank-6 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ,
                                                    Bnd b4 ,
                                                    Bnd b5 ,
                                                    Bnd b6 ,
                                                    Bnd b7 ) : Array(label,data,Bnds(b1,b2,b3,b4,b5,b6,b7)) {
      static_assert( rank == 7 , "ERROR: Calling constructor with 7 bound on non-rank-7 array" );
    }
    YAKL_INLINE Array( char const *label , T *data, Bnd b1 ,
                                                    Bnd b2 ,
                                                    Bnd b3 ,
                                                    Bnd b4 ,
                                                    Bnd b5 ,
                                                    Bnd b6 ,
                                                    Bnd b7 ,
                                                    Bnd b8 ) : Array(label,data,Bnds(b1,b2,b3,b4,b5,b6,b7,b8)) {
      static_assert( rank == 8 , "ERROR: Calling constructor with 8 bound on non-rank-8 array" );
    }
    YAKL_INLINE Array(char const *label, T *data, Bnds bnds) {
      static_assert( rank >= 1 && rank <= 8 , "ERROR: Creating Array with a rank < 1 or > 8" );
      nullify();
      #ifdef YAKL_DEBUG
        if ( bnds.size() < rank ) { yakl_throw("ERROR: Number of array bounds specified is < rank"); }
        if (data == nullptr) yakl_throw("ERROR: wrapping nullptr with a YAKL Array object");
        this->myname = label;
      #endif
      for (int i=0; i < rank; i++) { this->lbounds[i] = bnds[i].l; this->dimension[i] = bnds[i].u - bnds[i].l + 1; }
      this->myData = data;
      this->refCount = nullptr;
    }
 
 
    /*
    COPY CONSTRUCTORS / FUNCTIONS
    This shares the pointers with another Array and increments the refCounter
    */
    YAKL_INLINE Array(Array<non_const_value_type,rank,myMem,styleFortran> const &rhs) {
      // This is a constructor, so no need to deallocate
      nullify();
      copy_constructor_common(rhs);
    }
    YAKL_INLINE Array(Array<const_value_type,rank,myMem,styleFortran> const &rhs) {
      static_assert( std::is_const<T>::value , 
                     "ERROR: Cannot create non-const Array using const Array" );
      // This is a constructor, so no need to deallocate
      nullify();
      copy_constructor_common(rhs);
    }
 
 
    YAKL_INLINE Array & operator=(Array<non_const_value_type,rank,myMem,styleFortran> const &rhs) {
      if constexpr (! std::is_const<T>::value) {
        if (this == &rhs) { return *this; }
      }
      YAKL_EXECUTE_ON_HOST_ONLY( this->deallocate(); )
      copy_constructor_common(rhs);
      return *this;
    }
    YAKL_INLINE Array & operator=(Array<const_value_type,rank,myMem,styleFortran> const &rhs) {
      static_assert( std::is_const<T>::value , 
                     "ERROR: Cannot create non-const Array using const Array" );
      if constexpr (std::is_const<T>::value) {
        if (this == &rhs) { return *this; }
      }
      YAKL_EXECUTE_ON_HOST_ONLY( this->deallocate(); )
      copy_constructor_common(rhs);
      return *this;
    }
 
    template <class TLOC>
    YAKL_INLINE void copy_constructor_common(Array<TLOC,rank,myMem,styleFortran> const &rhs) {
      for (int i=0; i<rank; i++) {
        this->lbounds[i] = rhs.lbounds[i]; this->dimension[i] = rhs.dimension[i];
      }
      #ifdef YAKL_DEBUG
        this->myname = rhs.myname;
      #endif
      this->myData   = rhs.myData;
      YAKL_EXECUTE_ON_HOST_ONLY( yakl_mtx_lock(); )
      this->refCount = rhs.refCount;
      if (this->refCount != nullptr) {
        // YAKL_EXECUTE_ON_HOST_ONLY( (*(this->refCount))++; )  // This gives an nvc++ error
        YAKL_EXECUTE_ON_HOST_ONLY( { (*(this->refCount))++; } )  // This works around the nvc++ error
      }
      YAKL_EXECUTE_ON_HOST_ONLY( yakl_mtx_unlock(); )
    }
 
 
    /*
    MOVE CONSTRUCTORS
    This steals the pointers form the rhs instead of sharing and sets rhs pointers to nullptr.
    Therefore, no need to increment reference counter
    */
    YAKL_INLINE Array(Array &&rhs) {
      // This is a constructor, so no need to deallocate
      nullify();
      for (int i=0; i<rank; i++) {
        this->lbounds[i] = rhs.lbounds[i]; this->dimension[i] = rhs.dimension[i];
      }
      #ifdef YAKL_DEBUG
        this->myname = rhs.myname;
      #endif
      this->myData   = rhs.myData;
      rhs.myData   = nullptr;
 
      this->refCount = rhs.refCount;
      rhs.refCount = nullptr;
    }
 
 
    YAKL_INLINE Array& operator=(Array &&rhs) {
      if (this == &rhs) { return *this; }
      YAKL_EXECUTE_ON_HOST_ONLY( this->deallocate(); )
      for (int i=0; i<rank; i++) {
        this->lbounds  [i] = rhs.lbounds  [i]; this->dimension[i] = rhs.dimension[i];
      }
      #ifdef YAKL_DEBUG
        this->myname = rhs.myname;
      #endif
      this->myData   = rhs.myData;
      rhs.myData   = nullptr;
 
      this->refCount = rhs.refCount;
      rhs.refCount = nullptr;
 
      return *this;
    }
 
 
    /*
    DESTRUCTOR
    Decrement the refCounter, and if it's zero, deallocate and nullify.  
    */
    YAKL_INLINE ~Array() {
      YAKL_EXECUTE_ON_HOST_ONLY( this->deallocate(); )
    }
 
 
    Array( array_ir::ArrayIR<T,rank> const &ir , std::vector<int> lower_bounds = std::vector<int>() ) {
      nullify();
      if (myMem == memDevice && (! ir.data_valid_on_device())) yakl_throw("ERROR: wrapping non-device-valid ArrayIR with memDevice yakl::FArray");
      if (myMem == memHost   && (! ir.data_valid_on_host  ())) yakl_throw("ERROR: wrapping non-host-valid ArrayIR with memHost yakl::FArray");
      this->myData = ir.data();
      #ifdef YAKL_DEBUG
        this->myname = ir.label();
      #endif
      for (int i=0; i < rank; i++) { this->dimension[i] = ir.extent(rank-1-i); }
      if ( (! lower_bounds.empty()) && ( lower_bounds.size() != rank ) ) yakl_throw("ERROR: Passed lower bounds of the wrong rank");
      for (int i=0; i < rank; i++) { this->lbounds[i] = lower_bounds[i]; }
    }
 
 
    template <class TLOC = T>
    array_ir::ArrayIR<TLOC,rank> create_ArrayIR() const {
      std::array<size_t,rank> dimensions;
      for (int i=0; i < rank; i++) { dimensions[i] = this->dimension[rank-1-i]; }
      if (myMem == memHost) {
        return array_ir::ArrayIR<TLOC,rank>(const_cast<TLOC *>(this->myData),dimensions,array_ir::MEMORY_HOST,this->label());
      } else {
        #ifdef YAKL_MANAGED_MEMORY
          return array_ir::ArrayIR<TLOC,rank>(const_cast<TLOC *>(this->myData),dimensions,array_ir::MEMORY_SHARED,this->label());
        #else
          return array_ir::ArrayIR<TLOC,rank>(const_cast<TLOC *>(this->myData),dimensions,array_ir::MEMORY_DEVICE,this->label());
        #endif
      }
    }
 
 
    // Common detailed documentation for all indexers
    /* ARRAY INDEXERS (FORTRAN index ordering)
    Return the element at the given index (either read-only or read-write)
    */
    YAKL_INLINE T &operator()(int i0) const {
      static_assert( rank == 1 , "ERROR: Indexing non-rank-1 array with 1 index" );
      #ifdef YAKL_DEBUG
        check(i0);
      #endif
      index_t ind = (i0-this->lbounds[0]);
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1) const {
      static_assert( rank == 2 , "ERROR: Indexing non-rank-2 array with 2 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1);
      #endif
      index_t ind =                       (i1-this->lbounds[1])  *
                     this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2) const {
      static_assert( rank == 3 , "ERROR: Indexing non-rank-3 array with 3 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2);
      #endif
      index_t ind = (                      (i2-this->lbounds[2])  *
                      this->dimension[1] + (i1-this->lbounds[1]) )*
                      this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2, int i3) const {
      static_assert( rank == 4 , "ERROR: Indexing non-rank-4 array with 4 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2,i3);
      #endif
      index_t ind = ((                      (i3-this->lbounds[3])  *
                       this->dimension[2] + (i2-this->lbounds[2]) )*
                       this->dimension[1] + (i1-this->lbounds[1]) )*
                       this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2, int i3, int i4) const {
      static_assert( rank == 5 , "ERROR: Indexing non-rank-5 array with 5 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2,i3,i4);
      #endif
      index_t ind = (((                      (i4-this->lbounds[4])  *
                        this->dimension[3] + (i3-this->lbounds[3]) )*
                        this->dimension[2] + (i2-this->lbounds[2]) )*
                        this->dimension[1] + (i1-this->lbounds[1]) )*
                        this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2, int i3, int i4, int i5) const {
      static_assert( rank == 6 , "ERROR: Indexing non-rank-6 array with 6 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2,i3,i4,i5);
      #endif
      index_t ind = ((((                      (i5-this->lbounds[5])  *
                         this->dimension[4] + (i4-this->lbounds[4]) )*
                         this->dimension[3] + (i3-this->lbounds[3]) )*
                         this->dimension[2] + (i2-this->lbounds[2]) )*
                         this->dimension[1] + (i1-this->lbounds[1]) )*
                         this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2, int i3, int i4, int i5, int i6) const {
      static_assert( rank == 7 , "ERROR: Indexing non-rank-7 array with 7 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2,i3,i4,i5,i6);
      #endif
      index_t ind = (((((                      (i6-this->lbounds[6])  *
                          this->dimension[5] + (i5-this->lbounds[5]) )*
                          this->dimension[4] + (i4-this->lbounds[4]) )*
                          this->dimension[3] + (i3-this->lbounds[3]) )*
                          this->dimension[2] + (i2-this->lbounds[2]) )*
                          this->dimension[1] + (i1-this->lbounds[1]) )*
                          this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
    YAKL_INLINE T &operator()(int i0, int i1, int i2, int i3, int i4, int i5, int i6, int i7) const {
      static_assert( rank == 8 , "ERROR: Indexing non-rank-8 array with 8 indices" );
      #ifdef YAKL_DEBUG
        check(i0,i1,i2,i3,i4,i5,i6,i7);
      #endif
      index_t ind = ((((((                      (i7-this->lbounds[7])  *
                           this->dimension[6] + (i6-this->lbounds[6]) )*
                           this->dimension[5] + (i5-this->lbounds[5]) )*
                           this->dimension[4] + (i4-this->lbounds[4]) )*
                           this->dimension[3] + (i3-this->lbounds[3]) )*
                           this->dimension[2] + (i2-this->lbounds[2]) )*
                           this->dimension[1] + (i1-this->lbounds[1]) )*
                           this->dimension[0] + (i0-this->lbounds[0]) ;
      return this->myData[ind];
    }
 
    
    YAKL_INLINE void check(int i0, int i1=1, int i2=1, int i3=1, int i4=1, int i5=1,
                           int i6=1, int i7=1) const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) { yakl_throw("Error: Using operator() on an Array that isn't allocated"); }
        if constexpr (rank >= 1) { if (i0 < this->lbounds[0] || i0 >= this->lbounds[0]+this->dimension[0]) ind_out_bounds<0>(i0); }
        if constexpr (rank >= 2) { if (i1 < this->lbounds[1] || i1 >= this->lbounds[1]+this->dimension[1]) ind_out_bounds<1>(i1); }
        if constexpr (rank >= 3) { if (i2 < this->lbounds[2] || i2 >= this->lbounds[2]+this->dimension[2]) ind_out_bounds<2>(i2); }
        if constexpr (rank >= 4) { if (i3 < this->lbounds[3] || i3 >= this->lbounds[3]+this->dimension[3]) ind_out_bounds<3>(i3); }
        if constexpr (rank >= 5) { if (i4 < this->lbounds[4] || i4 >= this->lbounds[4]+this->dimension[4]) ind_out_bounds<4>(i4); }
        if constexpr (rank >= 6) { if (i5 < this->lbounds[5] || i5 >= this->lbounds[5]+this->dimension[5]) ind_out_bounds<5>(i5); }
        if constexpr (rank >= 7) { if (i6 < this->lbounds[6] || i6 >= this->lbounds[6]+this->dimension[6]) ind_out_bounds<6>(i6); }
        if constexpr (rank >= 8) { if (i7 < this->lbounds[7] || i7 >= this->lbounds[7]+this->dimension[7]) ind_out_bounds<7>(i7); }
        #if defined(YAKL_SEPARATE_MEMORY_SPACE)
          YAKL_EXECUTE_ON_DEVICE_ONLY( if constexpr (myMem == memHost) yakl_throw("ERROR: host array being accessed in a device kernel"); )
          #if !defined(YAKL_MANAGED_MEMORY)
            YAKL_EXECUTE_ON_HOST_ONLY(
              if constexpr (myMem == memDevice) {
                std::cerr << "ERROR: For Array labeled: " << this->myname << ":" << std::endl;
                yakl_throw("Device array being accessed on the host without managed memory turned on");
              }
            )
          #endif
        #endif
      #endif
    }
 
 
    // if this function gets called, then there was definitely an error
    template <int I>
    YAKL_INLINE void ind_out_bounds(int ind) const {
      #ifdef YAKL_DEBUG
        YAKL_EXECUTE_ON_HOST_ONLY(
          std::cerr << "ERROR: For Array labeled: " << this->myname << ":" << std::endl;
          std::cerr << "Index " << I+1 << " of " << rank << " is out of bounds.  Provided index: " << ind
                    << ".  Lower Bound: " << this->lbounds[I] << ".  Upper Bound: " << this->dimension[I]-1 << std::endl;
        )
        yakl_throw("ERROR: Index out of bounds.");
      #endif
    }
 
 
    template <class TLOC, typename std::enable_if<std::is_arithmetic<TLOC>::value,bool>::type = false>
    Array operator=(TLOC const &rhs) const {
      memset_loc(rhs);
      return *this;
    }
 
 
    template <class TLOC>
    void memset_loc(TLOC rhs) const {
      if (myMem == memDevice) {
        #ifdef YAKL_ENABLE_STREAMS
          fence();
        #endif
        memset(*this, rhs);
        #ifdef YAKL_ENABLE_STREAMS
          fence();
        #endif
      } else {
        for (int i=0; i < this->totElems(); i++) { this->myData[i] = rhs; }
      }
    }
 
 
    YAKL_INLINE Array<T,rank,myMem,styleFortran> subset_slowest_dimension(int u) const { return subset_slowest_dimension(lbounds[rank-1],u); }
 
 
    YAKL_INLINE Array<T,rank,myMem,styleFortran> subset_slowest_dimension(int l, int u) const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) { yakl_throw("ERROR: Trying to subset_slowest_dimension an Array that hasn't been initialized"); }
        if (l < lbounds[rank-1]) { yakl_throw("ERROR: subset_slowest_dimension lower bound too low"); }
        if (u < lbounds[rank-1]) { yakl_throw("ERROR: subset_slowest_dimension upper bound too low"); }
        if (l > u) { yakl_throw("ERROR: subset_slowest_dimension lower bound > upper bounds"); }
        if (u >= this->lbounds[rank-1]+this->dimension[rank-1]) { yakl_throw("ERROR: subset_slowest_dimension upper bound too high"); }
      #endif
      auto ret = *this;
      auto &lb = this->lbounds;
      auto &d  = this->dimension;
      if constexpr (rank == 1) ret.myData += (l-lb[rank-1]);
      if constexpr (rank == 2) ret.myData += (l-lb[rank-1])*d[0];
      if constexpr (rank == 3) ret.myData += (l-lb[rank-1])*d[0]*d[1];
      if constexpr (rank == 4) ret.myData += (l-lb[rank-1])*d[0]*d[1]*d[2];
      if constexpr (rank == 5) ret.myData += (l-lb[rank-1])*d[0]*d[1]*d[2]*d[3];
      if constexpr (rank == 6) ret.myData += (l-lb[rank-1])*d[0]*d[1]*d[2]*d[3]*d[4];
      if constexpr (rank == 7) ret.myData += (l-lb[rank-1])*d[0]*d[1]*d[2]*d[3]*d[4]*d[5];
      if constexpr (rank == 8) ret.myData += (l-lb[rank-1])*d[0]*d[1]*d[2]*d[3]*d[4]*d[5]*d[6];
      ret.dimension[rank-1] = u-l+1;
      return ret;
    }
 
 
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( Dims const &dims ) const {
      #ifdef YAKL_DEBUG
        if (rank != dims.size()) {
          YAKL_EXECUTE_ON_HOST_ONLY( std::cerr << "For Array named " << this->myname << ":  "; )
          yakl_throw("ERROR: rank must be equal to dims.size()");
        }
        for (int i=N; i<rank; i++) {
          if (dims.data[i] < this->lbounds[i] || dims.data[i] >= this->lbounds[i]+this->dimension[i] ) {
            YAKL_EXECUTE_ON_HOST_ONLY( std::cerr << "For Array named " << this->myname << ":  "; )
            yakl_throw("ERROR: One of the slicing dimension dimensions is out of bounds");
          }
        }
        if (! this->initialized()) {
          YAKL_EXECUTE_ON_HOST_ONLY( std::cerr << "For Array named " << this->myname << ":  "; )
          yakl_throw("ERROR: calling slice() on an Array that hasn't been allocated");
        }
      #endif
      Array<T,N,myMem,styleFortran> ret;
      index_t offset = 1;
      for (int i=0; i<N; i++) {
        ret.dimension[i] = this->dimension[i];
        ret.lbounds  [i] = this->lbounds  [i];
        offset *= this->dimension[i];
      }
      index_t retOff = 0;
      for (int i=N; i<rank; i++) {
        retOff += (dims.data[i]-this->lbounds[i])*offset;
        offset *= this->dimension[i];
      }
      ret.myData = &(this->myData[retOff]);
      YAKL_EXECUTE_ON_HOST_ONLY(
        yakl_mtx_lock();
        ret.refCount = this->refCount;
        if (this->refCount != nullptr) {
          (*(this->refCount))++;
        }
        yakl_mtx_unlock();
      )
      return ret;
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0 ) const {
      static_assert( rank == 1 , "ERROR: Calling slice() with 1 index on a non-rank-1 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1 ) const {
      static_assert( rank == 2 , "ERROR: Calling slice() with 2 index on a non-rank-2 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2 ) const {
      static_assert( rank == 3 , "ERROR: Calling slice() with 3 index on a non-rank-3 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2, int i3 ) const {
      static_assert( rank == 4 , "ERROR: Calling slice() with 4 index on a non-rank-4 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2,i3) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2, int i3,
                                                                      int i4 ) const {
      static_assert( rank == 5 , "ERROR: Calling slice() with 5 index on a non-rank-5 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2,i3,i4) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2, int i3,
                                                                      int i4, int i5 ) const {
      static_assert( rank == 6 , "ERROR: Calling slice() with 6 index on a non-rank-6 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2,i3,i4,i5) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2, int i3,
                                                                      int i4, int i5, int i6 ) const {
      static_assert( rank == 7 , "ERROR: Calling slice() with 7 index on a non-rank-7 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2,i3,i4,i5,i6) );
    }
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> slice( int i0, int i1, int i2, int i3,
                                                                      int i4, int i5, int i6, int i7 ) const {
      static_assert( rank == 8 , "ERROR: Calling slice() with 8 index on a non-rank-8 array" );
      static_assert( N <= rank , "ERROR: Calling slice() with more dimenions than this array's rank" );
      return slice<N>( Dims(i0,i1,i2,i3,i4,i5,i6,i7) );
    }
 
 
 
    template <int N> YAKL_INLINE Array<T,N,myMem,styleFortran> reshape(Bnds const &bnds) const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) { yakl_throw("ERROR: Trying to reshape an Array that hasn't been initialized"); }
        if (bnds.size() != N) { yakl_throw("ERROR: new number of reshaped array dimensions does not match the templated rank"); }
        index_t totelems = 1;
        for (int i=0; i < N; i++) { totelems *= (bnds.u[i]-bnds.l[i]+1); }
        if (totelems != this->totElems()) { yakl_throw("ERROR: Total number of reshaped array elements is not consistent with this array"); }
      #endif
      Array<T,N,myMem,styleFortran> ret;
      for (int i=0; i < N; i++) {
        ret.dimension[i] = bnds.u[i] - bnds.l[i] + 1;  ret.lbounds  [i] = bnds.l[i];
      }
      #ifdef YAKL_DEBUG
        ret.myname = this->myname;
      #endif
      ret.myData = this->myData;
      YAKL_EXECUTE_ON_HOST_ONLY(
        yakl_mtx_lock();
        ret.refCount = this->refCount;
        if (this->refCount != nullptr) {
          (*(this->refCount))++;
        }
        yakl_mtx_unlock();
      )
      return ret;
    }
    YAKL_INLINE Array<T,1,myMem,styleFortran> reshape(Bnd b0                                                        ) const { return reshape<1>( Bnds(b0) ); }
    YAKL_INLINE Array<T,2,myMem,styleFortran> reshape(Bnd b0, Bnd b1                                                ) const { return reshape<2>( Bnds(b0,b1) ); }
    YAKL_INLINE Array<T,3,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2                                        ) const { return reshape<3>( Bnds(b0,b1,b2) ); }
    YAKL_INLINE Array<T,4,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2, Bnd b3                                ) const { return reshape<4>( Bnds(b0,b1,b2,b3) ); }
    YAKL_INLINE Array<T,5,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2, Bnd b3, Bnd b4                        ) const { return reshape<5>( Bnds(b0,b1,b2,b3,b4) ); }
    YAKL_INLINE Array<T,6,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2, Bnd b3, Bnd b4, Bnd b5                ) const { return reshape<6>( Bnds(b0,b1,b2,b3,b4,b5) ); }
    YAKL_INLINE Array<T,7,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2, Bnd b3, Bnd b4, Bnd b5, Bnd b6        ) const { return reshape<7>( Bnds(b0,b1,b2,b3,b4,b5,b6) ); }
    YAKL_INLINE Array<T,8,myMem,styleFortran> reshape(Bnd b0, Bnd b1, Bnd b2, Bnd b3, Bnd b4, Bnd b5, Bnd b6, Bnd b7) const { return reshape<8>( Bnds(b0,b1,b2,b3,b4,b5,b6,b7) ); }
 
 
    YAKL_INLINE Array<T,1,myMem,styleFortran> collapse(int lbnd=1) const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) { yakl_throw("ERROR: Trying to collapse an Array that hasn't been initialized"); }
      #endif
      Array<T,1,myMem,styleFortran> ret;
      ret.dimension[0] = this->totElems();  ret.lbounds  [0] = lbnd;
      #ifdef YAKL_DEBUG
        ret.myname = this->myname;
      #endif
      ret.myData = this->myData;
      YAKL_EXECUTE_ON_HOST_ONLY(
        yakl_mtx_lock();
        ret.refCount = this->refCount;
        if (this->refCount != nullptr) {
          (*(this->refCount))++;
        }
        yakl_mtx_unlock();
      )
      return ret;
    }
 
 
    // Create a host copy of this array. Even if the array exists on the host, a deep copy to a separate
    // object is still performed to avoid any potential bugs when the user expects this behavior
    template <class TLOC=T>
    inline Array<TLOC,rank,memHost,styleFortran> createHostCopy(Stream stream = Stream()) const {
      auto ret = createHostObject();
      this->copy_inform(ret);
      if (myMem == memHost) { memcpy_host_to_host  ( ret.myData , this->myData , this->totElems()          ); }
      else                  { memcpy_device_to_host( ret.myData , this->myData , this->totElems() , stream ); }
      if (stream.is_default_stream()) { fence(); }
      else                            { stream.fence(); }
      return Array<TLOC,rank,memHost,styleFortran>(ret);
    }
 
 
    // Create a separate host Array with the same rank memory space and style
    template <class TLOC=typename std::remove_cv<T>::type>
    inline Array<typename std::remove_cv<TLOC>::type,rank,memHost,styleFortran> createHostObject() const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) {
          YAKL_EXECUTE_ON_HOST_ONLY( std::cerr << "For Array named " << this->myname << ":  "; )
          yakl_throw("Error: createHostCopy() called on an Array that hasn't been allocated.");
        }
      #endif
      // If this Array is of const type, then we need to use non-const when allocating, then cast it to const aterward
      Array<typename std::remove_cv<TLOC>::type,rank,memHost,styleFortran> ret;  // nullified + owned == true
      for (int i=0; i<rank; i++) { ret.lbounds[i] = this->lbounds[i];  ret.dimension[i] = this->dimension[i]; }
      #ifdef YAKL_DEBUG
        ret.myname = this->myname;
      #endif
      ret.allocate();
      return ret;
    }
 
 
    // Create a device copy of this array. Even if the array exists on the host, a deep copy to a separate
    // object is still performed to avoid any potential bugs when the user expects this behavior
    template <class TLOC=T>
    inline Array<TLOC,rank,memDevice,styleFortran> createDeviceCopy(Stream stream = Stream()) const {
      auto ret = createDeviceObject();
      this->copy_inform(ret);
      if (myMem == memHost) { memcpy_host_to_device  ( ret.myData , this->myData , this->totElems() , stream ); }
      else                  { memcpy_device_to_device( ret.myData , this->myData , this->totElems() , stream ); }
      if (stream.is_default_stream()) { fence(); }
      else                            { stream.fence(); }
      return Array<TLOC,rank,memDevice,styleFortran>(ret);
    }
 
 
    // Create separate device array with the same rank, memory space, and style
    template <class TLOC=typename std::remove_cv<T>::type>
    inline Array<typename std::remove_cv<TLOC>::type,rank,memDevice,styleFortran> createDeviceObject() const {
      #ifdef YAKL_DEBUG
        if (! this->initialized()) {
          YAKL_EXECUTE_ON_HOST_ONLY( std::cerr << "For Array named " << this->myname << ":  "; )
          yakl_throw("Error: createHostCopy() called on an Array that hasn't been allocated.");
        }
      #endif
      // If this Array is of const type, then we need to use non-const when allocating, then cast it to const aterward
      Array<typename std::remove_cv<TLOC>::type,rank,memDevice,styleFortran> ret;  // nullified + owned == true
      for (int i=0; i<rank; i++) { ret.lbounds[i] = this->lbounds[i];  ret.dimension[i] = this->dimension[i]; }
      #ifdef YAKL_DEBUG
        ret.myname = this->myname;
      #endif
      ret.allocate();
      return ret;
    }
 
 
    /* ACCESSORS */
    YAKL_INLINE FSArray<index_t,1,SB<rank>> get_dimensions() const {
      FSArray<index_t,1,SB<rank>> ret;
      for (int i=0; i<rank; i++) { ret(i+1) = this->dimension[i]; }
      return ret;
    }
    YAKL_INLINE FSArray<int,1,SB<rank>> get_lbounds() const {
      FSArray<int,1,SB<rank>> ret;
      for (int i=0; i<rank; i++) { ret(i+1) = this->lbounds[i]; }
      return ret;
    }
    YAKL_INLINE FSArray<int,1,SB<rank>> get_ubounds() const {
      FSArray<int,1,SB<rank>> ret;
      for (int i=0; i<rank; i++) { ret(i+1) = this->lbounds[i]+this->dimension[i]-1; }
      return ret;
    }
    YAKL_INLINE index_t extent( int dim ) const {
      #ifdef YAKL_DEBUG
        if (dim < 0 || dim > rank-1) yakl_throw("ERROR: calling extent() with an out of bounds index");
      #endif
      return this->dimension[dim];
    }
 
  };
 
}
__YAKL_NAMESPACE_WRAPPER_END__