Blaze  3.6
DMatTSMatMultExpr.h
Go to the documentation of this file.
1 //=================================================================================================
33 //=================================================================================================
34 
35 #ifndef _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
37 
38 
39 //*************************************************************************************************
40 // Includes
41 //*************************************************************************************************
42 
43 #include <blaze/math/Aliases.h>
53 #include <blaze/math/Exception.h>
65 #include <blaze/math/shims/Reset.h>
88 #include <blaze/math/views/Check.h>
93 #include <blaze/util/Assert.h>
94 #include <blaze/util/DisableIf.h>
95 #include <blaze/util/EnableIf.h>
98 #include <blaze/util/MaybeUnused.h>
99 #include <blaze/util/mpl/If.h>
100 #include <blaze/util/Types.h>
103 
104 
105 namespace blaze {
106 
107 //=================================================================================================
108 //
109 // CLASS DMATTSMATMULTEXPR
110 //
111 //=================================================================================================
112 
113 //*************************************************************************************************
120 template< typename MT1 // Type of the left-hand side dense matrix
121  , typename MT2 // Type of the right-hand side sparse matrix
122  , bool SF // Symmetry flag
123  , bool HF // Hermitian flag
124  , bool LF // Lower flag
125  , bool UF > // Upper flag
127  : public MatMatMultExpr< DenseMatrix< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, false > >
128  , private Computation
129 {
130  private:
131  //**Type definitions****************************************************************************
138  //**********************************************************************************************
139 
140  //**********************************************************************************************
142  static constexpr bool evaluateLeft = ( IsComputation_v<MT1> || RequiresEvaluation_v<MT1> );
143  //**********************************************************************************************
144 
145  //**********************************************************************************************
147  static constexpr bool evaluateRight = ( IsComputation_v<MT2> || RequiresEvaluation_v<MT2> );
148  //**********************************************************************************************
149 
150  //**********************************************************************************************
151  static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
152  static constexpr bool HERM = ( HF && !( LF || UF ) );
153  static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
154  static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
155  //**********************************************************************************************
156 
157  //**********************************************************************************************
159 
164  template< typename T1, typename T2, typename T3 >
165  static constexpr bool CanExploitSymmetry_v = IsSymmetric_v<T2>;
167  //**********************************************************************************************
168 
169  //**********************************************************************************************
171 
175  template< typename T1, typename T2, typename T3 >
176  static constexpr bool IsEvaluationRequired_v =
177  ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
179  //**********************************************************************************************
180 
181  //**********************************************************************************************
183 
186  template< typename T1, typename T2, typename T3 >
187  static constexpr bool UseOptimizedKernel_v =
188  ( useOptimizedKernels &&
189  !IsDiagonal_v<T2> &&
190  !IsResizable_v< ElementType_t<T1> > &&
191  !IsResizable_v<ET2> );
193  //**********************************************************************************************
194 
195  //**********************************************************************************************
197 
200  using ForwardFunctor = If_t< HERM
201  , DeclHerm
202  , If_t< SYM
203  , DeclSym
204  , If_t< LOW
205  , If_t< UPP
206  , DeclDiag
207  , DeclLow >
208  , If_t< UPP
209  , DeclUpp
210  , Noop > > > >;
212  //**********************************************************************************************
213 
214  public:
215  //**Type definitions****************************************************************************
218 
221 
223  using ResultType = typename If_t< HERM
225  , If_t< SYM
227  , If_t< LOW
228  , If_t< UPP
231  , If_t< UPP
233  , MultTrait<RT1,RT2> > > > >::Type;
234 
238  using ReturnType = const ElementType;
239  using CompositeType = const ResultType;
240 
242  using LeftOperand = If_t< IsExpression_v<MT1>, const MT1, const MT1& >;
243 
245  using RightOperand = If_t< IsExpression_v<MT2>, const MT2, const MT2& >;
246 
249 
252  //**********************************************************************************************
253 
254  //**Compilation flags***************************************************************************
256  static constexpr bool simdEnabled = false;
257 
259  static constexpr bool smpAssignable =
260  ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
261  //**********************************************************************************************
262 
263  //**Constructor*********************************************************************************
269  explicit inline DMatTSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
270  : lhs_( lhs ) // Left-hand side dense matrix of the multiplication expression
271  , rhs_( rhs ) // Right-hand side sparse matrix of the multiplication expression
272  {
273  BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
274  }
275  //**********************************************************************************************
276 
277  //**Access operator*****************************************************************************
284  inline ReturnType operator()( size_t i, size_t j ) const {
285  BLAZE_INTERNAL_ASSERT( i < lhs_.rows() , "Invalid row access index" );
286  BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
287 
288  if( IsDiagonal_v<MT1> ) {
289  return lhs_(i,i) * rhs_(i,j);
290  }
291  else if( IsDiagonal_v<MT2> ) {
292  return lhs_(i,j) * rhs_(j,j);
293  }
294  else if( IsTriangular_v<MT1> || IsTriangular_v<MT2> ) {
295  const size_t begin( ( IsUpper_v<MT1> )
296  ?( ( IsLower_v<MT2> )
297  ?( max( ( IsStrictlyUpper_v<MT1> ? i+1UL : i )
298  , ( IsStrictlyLower_v<MT2> ? j+1UL : j ) ) )
299  :( IsStrictlyUpper_v<MT1> ? i+1UL : i ) )
300  :( ( IsLower_v<MT2> )
301  ?( IsStrictlyLower_v<MT2> ? j+1UL : j )
302  :( 0UL ) ) );
303  const size_t end( ( IsLower_v<MT1> )
304  ?( ( IsUpper_v<MT2> )
305  ?( min( ( IsStrictlyLower_v<MT1> ? i : i+1UL )
306  , ( IsStrictlyUpper_v<MT2> ? j : j+1UL ) ) )
307  :( IsStrictlyLower_v<MT1> ? i : i+1UL ) )
308  :( ( IsUpper_v<MT2> )
309  ?( IsStrictlyUpper_v<MT2> ? j : j+1UL )
310  :( lhs_.columns() ) ) );
311 
312  if( begin >= end ) return ElementType();
313 
314  const size_t n( end - begin );
315 
316  return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
317  subvector( column( rhs_, j, unchecked ), begin, n, unchecked );
318  }
319  else {
320  return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
321  }
322  }
323  //**********************************************************************************************
324 
325  //**At function*********************************************************************************
333  inline ReturnType at( size_t i, size_t j ) const {
334  if( i >= lhs_.rows() ) {
335  BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
336  }
337  if( j >= rhs_.columns() ) {
338  BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
339  }
340  return (*this)(i,j);
341  }
342  //**********************************************************************************************
343 
344  //**Rows function*******************************************************************************
349  inline size_t rows() const noexcept {
350  return lhs_.rows();
351  }
352  //**********************************************************************************************
353 
354  //**Columns function****************************************************************************
359  inline size_t columns() const noexcept {
360  return rhs_.columns();
361  }
362  //**********************************************************************************************
363 
364  //**Left operand access*************************************************************************
369  inline LeftOperand leftOperand() const noexcept {
370  return lhs_;
371  }
372  //**********************************************************************************************
373 
374  //**Right operand access************************************************************************
379  inline RightOperand rightOperand() const noexcept {
380  return rhs_;
381  }
382  //**********************************************************************************************
383 
384  //**********************************************************************************************
390  template< typename T >
391  inline bool canAlias( const T* alias ) const noexcept {
392  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
393  }
394  //**********************************************************************************************
395 
396  //**********************************************************************************************
402  template< typename T >
403  inline bool isAliased( const T* alias ) const noexcept {
404  return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
405  }
406  //**********************************************************************************************
407 
408  //**********************************************************************************************
413  inline bool isAligned() const noexcept {
414  return lhs_.isAligned();
415  }
416  //**********************************************************************************************
417 
418  //**********************************************************************************************
423  inline bool canSMPAssign() const noexcept {
424  return ( rows() * columns() >= SMP_DMATTSMATMULT_THRESHOLD ) && !IsDiagonal_v<MT1>;
425  }
426  //**********************************************************************************************
427 
428  private:
429  //**Member variables****************************************************************************
432  //**********************************************************************************************
433 
434  //**Assignment to dense matrices****************************************************************
447  template< typename MT // Type of the target dense matrix
448  , bool SO > // Storage order of the target dense matrix
449  friend inline auto assign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
451  {
453 
454  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
455  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
456 
457  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
458  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
459 
460  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
461  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
462  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
463 
464  DMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
465  }
467  //**********************************************************************************************
468 
469  //**Default assignment to dense matrices********************************************************
483  template< typename MT3 // Type of the left-hand side target matrix
484  , typename MT4 // Type of the left-hand side matrix operand
485  , typename MT5 > // Type of the right-hand side matrix operand
486  static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
488  {
489  const size_t M( A.rows() );
490  const size_t N( B.columns() );
491 
492  BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
493 
494  if( LOW && UPP ) {
495  reset( C );
496  }
497 
498  {
499  size_t i( 0UL );
500 
501  for( ; (i+4UL) <= M; i+=4UL ) {
502  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
503  {
504  auto element( ( IsUpper_v<MT4> )
505  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
506  :( B.begin(j) ) );
507  const auto end( ( IsLower_v<MT4> )
508  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
509  :( B.end(j) ) );
510 
511  if( element == end ) {
512  reset( C(i ,j) );
513  reset( C(i+1UL,j) );
514  reset( C(i+2UL,j) );
515  reset( C(i+3UL,j) );
516  continue;
517  }
518 
519  C(i ,j) = A(i ,element->index()) * element->value();
520  C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
521  C(i+2UL,j) = A(i+2UL,element->index()) * element->value();
522  C(i+3UL,j) = A(i+3UL,element->index()) * element->value();
523  ++element;
524  for( ; element!=end; ++element ) {
525  C(i ,j) += A(i ,element->index()) * element->value();
526  C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
527  C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
528  C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
529  }
530  }
531  }
532 
533  for( ; (i+2UL) <= M; i+=2UL ) {
534  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
535  {
536  auto element( ( IsUpper_v<MT4> )
537  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
538  :( B.begin(j) ) );
539  const auto end( ( IsLower_v<MT4> )
540  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
541  :( B.end(j) ) );
542 
543  if( element == end ) {
544  reset( C(i ,j) );
545  reset( C(i+1UL,j) );
546  continue;
547  }
548 
549  C(i ,j) = A(i ,element->index()) * element->value();
550  C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
551  ++element;
552  for( ; element!=end; ++element ) {
553  C(i ,j) += A(i ,element->index()) * element->value();
554  C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
555  }
556  }
557  }
558 
559  for( ; i<M; ++i ) {
560  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
561  {
562  auto element( ( IsUpper_v<MT4> )
563  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
564  :( B.begin(j) ) );
565  const auto end( ( IsLower_v<MT4> )
566  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
567  :( B.end(j) ) );
568 
569  if( element == end ) {
570  reset( C(i,j) );
571  continue;
572  }
573 
574  C(i,j) = A(i,element->index()) * element->value();
575  ++element;
576  for( ; element!=end; ++element )
577  C(i,j) += A(i,element->index()) * element->value();
578  }
579  }
580  }
581 
582  if( SYM || HERM ) {
583  for( size_t i=1UL; i<M; ++i ) {
584  for( size_t j=0UL; j<i; ++j ) {
585  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
586  }
587  }
588  }
589  else if( LOW && !UPP ) {
590  for( size_t j=1UL; j<N; ++j ) {
591  for( size_t i=0UL; i<j; ++i ) {
592  reset( C(i,j) );
593  }
594  }
595  }
596  else if( !LOW && UPP ) {
597  for( size_t i=1UL; i<M; ++i ) {
598  for( size_t j=0UL; j<i; ++j ) {
599  reset( C(i,j) );
600  }
601  }
602  }
603  }
605  //**********************************************************************************************
606 
607  //**Optimized assignment to dense matrices******************************************************
621  template< typename MT3 // Type of the left-hand side target matrix
622  , typename MT4 // Type of the left-hand side matrix operand
623  , typename MT5 > // Type of the right-hand side matrix operand
624  static inline auto selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
625  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
626  {
627  const size_t M( A.rows() );
628  const size_t N( B.columns() );
629 
630  BLAZE_INTERNAL_ASSERT( !( SYM || HERM || LOW || UPP ) || M == N, "Broken invariant detected" );
631 
632  reset( C );
633 
634  {
635  size_t i( 0UL );
636 
637  for( ; (i+4UL) <= M; i+=4UL ) {
638  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
639  {
640  auto element( ( IsUpper_v<MT4> )
641  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
642  :( B.begin(j) ) );
643  const auto end( ( IsLower_v<MT4> )
644  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
645  :( B.end(j) ) );
646 
647  const size_t nonzeros( end - element );
648  const size_t kpos( nonzeros & size_t(-4) );
649  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
650 
651  for( size_t k=0UL; k<kpos; k+=4UL )
652  {
653  const size_t j1( element->index() );
654  const ET2 v1( element->value() );
655  ++element;
656  const size_t j2( element->index() );
657  const ET2 v2( element->value() );
658  ++element;
659  const size_t j3( element->index() );
660  const ET2 v3( element->value() );
661  ++element;
662  const size_t j4( element->index() );
663  const ET2 v4( element->value() );
664  ++element;
665 
666  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
667 
668  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
669  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
670  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
671  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
672  }
673 
674  for( ; element!=end; ++element )
675  {
676  const size_t j1( element->index() );
677  const ET2 v1( element->value() );
678 
679  C(i ,j) += A(i ,j1) * v1;
680  C(i+1UL,j) += A(i+1UL,j1) * v1;
681  C(i+2UL,j) += A(i+2UL,j1) * v1;
682  C(i+3UL,j) += A(i+3UL,j1) * v1;
683  }
684  }
685  }
686 
687  for( ; (i+2UL) <= M; i+=2UL ) {
688  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
689  {
690  auto element( ( IsUpper_v<MT4> )
691  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
692  :( B.begin(j) ) );
693  const auto end( ( IsLower_v<MT4> )
694  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
695  :( B.end(j) ) );
696 
697  const size_t nonzeros( end - element );
698  const size_t kpos( nonzeros & size_t(-4) );
699  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
700 
701  for( size_t k=0UL; k<kpos; k+=4UL )
702  {
703  const size_t j1( element->index() );
704  const ET2 v1( element->value() );
705  ++element;
706  const size_t j2( element->index() );
707  const ET2 v2( element->value() );
708  ++element;
709  const size_t j3( element->index() );
710  const ET2 v3( element->value() );
711  ++element;
712  const size_t j4( element->index() );
713  const ET2 v4( element->value() );
714  ++element;
715 
716  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
717 
718  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
719  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
720  }
721 
722  for( ; element!=end; ++element )
723  {
724  const size_t j1( element->index() );
725  const ET2 v1( element->value() );
726 
727  C(i ,j) += A(i ,j1) * v1;
728  C(i+1UL,j) += A(i+1UL,j1) * v1;
729  }
730  }
731  }
732 
733  for( ; i<M; ++i ) {
734  for( size_t j=( SYM || HERM || UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
735  {
736  auto element( ( IsUpper_v<MT4> )
737  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
738  :( B.begin(j) ) );
739  const auto end( ( IsLower_v<MT4> )
740  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
741  :( B.end(j) ) );
742 
743  const size_t nonzeros( end - element );
744  const size_t kpos( nonzeros & size_t(-4) );
745  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
746 
747  for( size_t k=0UL; k<kpos; k+=4UL )
748  {
749  const size_t j1( element->index() );
750  const ET2 v1( element->value() );
751  ++element;
752  const size_t j2( element->index() );
753  const ET2 v2( element->value() );
754  ++element;
755  const size_t j3( element->index() );
756  const ET2 v3( element->value() );
757  ++element;
758  const size_t j4( element->index() );
759  const ET2 v4( element->value() );
760  ++element;
761 
762  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
763 
764  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
765  }
766 
767  for( ; element!=end; ++element )
768  {
769  const size_t j1( element->index() );
770  const ET2 v1( element->value() );
771 
772  C(i,j) += A(i,j1) * v1;
773  }
774  }
775  }
776  }
777 
778  if( SYM || HERM ) {
779  for( size_t i=1UL; i<M; ++i ) {
780  for( size_t j=0UL; j<i; ++j ) {
781  C(i,j) = HERM ? conj( C(j,i) ) : C(j,i);
782  }
783  }
784  }
785  }
787  //**********************************************************************************************
788 
789  //**Assignment to sparse matrices***************************************************************
802  template< typename MT // Type of the target sparse matrix
803  , bool SO > // Storage order of the target sparse matrix
804  friend inline auto assign( SparseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
805  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
806  {
808 
809  using TmpType = If_t< SO, OppositeType, ResultType >;
810 
817 
818  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
819  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
820 
821  const ForwardFunctor fwd;
822 
823  const TmpType tmp( serial( rhs ) );
824  assign( ~lhs, fwd( tmp ) );
825  }
827  //**********************************************************************************************
828 
829  //**Restructuring assignment********************************************************************
844  template< typename MT // Type of the target matrix
845  , bool SO > // Storage order of the target matrix
846  friend inline auto assign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
847  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
848  {
850 
851  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
852  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
853 
854  const ForwardFunctor fwd;
855 
856  assign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
857  }
859  //**********************************************************************************************
860 
861  //**Addition assignment to dense matrices*******************************************************
874  template< typename MT // Type of the target dense matrix
875  , bool SO > // Storage order of the target dense matrix
876  friend inline auto addAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
877  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
878  {
880 
881  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
882  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
883 
884  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
885  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
886 
887  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
888  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
889  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
890 
891  DMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
892  }
894  //**********************************************************************************************
895 
896  //**Default addition assignment to dense matrices***********************************************
910  template< typename MT3 // Type of the left-hand side target matrix
911  , typename MT4 // Type of the left-hand side matrix operand
912  , typename MT5 > // Type of the right-hand side matrix operand
913  static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
914  -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
915  {
916  const size_t M( A.rows() );
917  const size_t N( B.columns() );
918 
919  BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
920 
921  {
922  size_t i( 0UL );
923 
924  for( ; (i+4UL) <= M; i+=4UL ) {
925  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
926  {
927  auto element( ( IsUpper_v<MT4> )
928  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
929  :( B.begin(j) ) );
930  const auto end( ( IsLower_v<MT4> )
931  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
932  :( B.end(j) ) );
933 
934  for( ; element!=end; ++element ) {
935  C(i ,j) += A(i ,element->index()) * element->value();
936  C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
937  C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
938  C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
939  }
940  }
941  }
942 
943  for( ; (i+2UL) <= M; i+=2UL ) {
944  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
945  {
946  auto element( ( IsUpper_v<MT4> )
947  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
948  :( B.begin(j) ) );
949  const auto end( ( IsLower_v<MT4> )
950  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
951  :( B.end(j) ) );
952 
953  for( ; element!=end; ++element ) {
954  C(i ,j) += A(i ,element->index()) * element->value();
955  C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
956  }
957  }
958  }
959 
960  for( ; i<M; ++i ) {
961  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
962  {
963  auto element( ( IsUpper_v<MT4> )
964  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
965  :( B.begin(j) ) );
966  const auto end( ( IsLower_v<MT4> )
967  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
968  :( B.end(j) ) );
969 
970  for( ; element!=end; ++element )
971  C(i,j) += A(i,element->index()) * element->value();
972  }
973  }
974  }
975  }
977  //**********************************************************************************************
978 
979  //**Optimized addition assignment to dense matrices*********************************************
993  template< typename MT3 // Type of the left-hand side target matrix
994  , typename MT4 // Type of the left-hand side matrix operand
995  , typename MT5 > // Type of the right-hand side matrix operand
996  static inline auto selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
997  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
998  {
999  const size_t M( A.rows() );
1000  const size_t N( B.columns() );
1001 
1002  BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1003 
1004  {
1005  size_t i( 0UL );
1006 
1007  for( ; (i+4UL) <= M; i+=4UL ) {
1008  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1009  {
1010  auto element( ( IsUpper_v<MT4> )
1011  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1012  :( B.begin(j) ) );
1013  const auto end( ( IsLower_v<MT4> )
1014  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1015  :( B.end(j) ) );
1016 
1017  const size_t nonzeros( end - element );
1018  const size_t kpos( nonzeros & size_t(-4) );
1019  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1020 
1021  for( size_t k=0UL; k<kpos; k+=4UL )
1022  {
1023  const size_t j1( element->index() );
1024  const ET2 v1( element->value() );
1025  ++element;
1026  const size_t j2( element->index() );
1027  const ET2 v2( element->value() );
1028  ++element;
1029  const size_t j3( element->index() );
1030  const ET2 v3( element->value() );
1031  ++element;
1032  const size_t j4( element->index() );
1033  const ET2 v4( element->value() );
1034  ++element;
1035 
1036  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1037 
1038  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1039  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1040  C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1041  C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1042  }
1043 
1044  for( ; element!=end; ++element )
1045  {
1046  const size_t j1( element->index() );
1047  const ET2 v1( element->value() );
1048 
1049  C(i ,j) += A(i ,j1) * v1;
1050  C(i+1UL,j) += A(i+1UL,j1) * v1;
1051  C(i+2UL,j) += A(i+2UL,j1) * v1;
1052  C(i+3UL,j) += A(i+3UL,j1) * v1;
1053  }
1054  }
1055  }
1056 
1057  for( ; (i+2UL) <= M; i+=2UL ) {
1058  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1059  {
1060  auto element( ( IsUpper_v<MT4> )
1061  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1062  :( B.begin(j) ) );
1063  const auto end( ( IsLower_v<MT4> )
1064  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1065  :( B.end(j) ) );
1066 
1067  const size_t nonzeros( end - element );
1068  const size_t kpos( nonzeros & size_t(-4) );
1069  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1070 
1071  for( size_t k=0UL; k<kpos; k+=4UL )
1072  {
1073  const size_t j1( element->index() );
1074  const ET2 v1( element->value() );
1075  ++element;
1076  const size_t j2( element->index() );
1077  const ET2 v2( element->value() );
1078  ++element;
1079  const size_t j3( element->index() );
1080  const ET2 v3( element->value() );
1081  ++element;
1082  const size_t j4( element->index() );
1083  const ET2 v4( element->value() );
1084  ++element;
1085 
1086  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1087 
1088  C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1089  C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1090  }
1091 
1092  for( ; element!=end; ++element )
1093  {
1094  const size_t j1( element->index() );
1095  const ET2 v1( element->value() );
1096 
1097  C(i ,j) += A(i ,j1) * v1;
1098  C(i+1UL,j) += A(i+1UL,j1) * v1;
1099  }
1100  }
1101  }
1102 
1103  for( ; i<M; ++i ) {
1104  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1105  {
1106  auto element( ( IsUpper_v<MT4> )
1107  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1108  :( B.begin(j) ) );
1109  const auto end( ( IsLower_v<MT4> )
1110  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1111  :( B.end(j) ) );
1112 
1113  const size_t nonzeros( end - element );
1114  const size_t kpos( nonzeros & size_t(-4) );
1115  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1116 
1117  for( size_t k=0UL; k<kpos; k+=4UL )
1118  {
1119  const size_t j1( element->index() );
1120  const ET2 v1( element->value() );
1121  ++element;
1122  const size_t j2( element->index() );
1123  const ET2 v2( element->value() );
1124  ++element;
1125  const size_t j3( element->index() );
1126  const ET2 v3( element->value() );
1127  ++element;
1128  const size_t j4( element->index() );
1129  const ET2 v4( element->value() );
1130  ++element;
1131 
1132  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1133 
1134  C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1135  }
1136 
1137  for( ; element!=end; ++element )
1138  {
1139  const size_t j1( element->index() );
1140  const ET2 v1( element->value() );
1141 
1142  C(i,j) += A(i,j1) * v1;
1143  }
1144  }
1145  }
1146  }
1147  }
1149  //**********************************************************************************************
1150 
1151  //**Restructuring addition assignment***********************************************************
1166  template< typename MT // Type of the target matrix
1167  , bool SO > // Storage order of the target matrix
1168  friend inline auto addAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1169  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1170  {
1172 
1174 
1175  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1176  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1177 
1178  const ForwardFunctor fwd;
1179 
1180  addAssign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1181  }
1183  //**********************************************************************************************
1184 
1185  //**Addition assignment to sparse matrices******************************************************
1186  // No special implementation for the addition assignment to sparse matrices.
1187  //**********************************************************************************************
1188 
1189  //**Subtraction assignment to dense matrices****************************************************
1202  template< typename MT // Type of the target dense matrix
1203  , bool SO > // Storage order of the target dense matrix
1204  friend inline auto subAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1205  -> DisableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1206  {
1208 
1209  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1210  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1211 
1212  LT A( serial( rhs.lhs_ ) ); // Evaluation of the left-hand side dense matrix operand
1213  RT B( serial( rhs.rhs_ ) ); // Evaluation of the right-hand side sparse matrix operand
1214 
1215  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1216  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1217  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
1218 
1219  DMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1220  }
1222  //**********************************************************************************************
1223 
1224  //**Default subtraction assignment to dense matrices********************************************
1238  template< typename MT3 // Type of the left-hand side target matrix
1239  , typename MT4 // Type of the left-hand side matrix operand
1240  , typename MT5 > // Type of the right-hand side matrix operand
1241  static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1242  -> DisableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1243  {
1244  const size_t M( A.rows() );
1245  const size_t N( B.columns() );
1246 
1247  BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1248 
1249  {
1250  size_t i( 0UL );
1251 
1252  for( ; (i+4UL) <= M; i+=4UL ) {
1253  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1254  {
1255  auto element( ( IsUpper_v<MT4> )
1256  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1257  :( B.begin(j) ) );
1258  const auto end( ( IsLower_v<MT4> )
1259  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1260  :( B.end(j) ) );
1261 
1262  for( ; element!=end; ++element ) {
1263  C(i ,j) -= A(i ,element->index()) * element->value();
1264  C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1265  C(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
1266  C(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
1267  }
1268  }
1269  }
1270 
1271  for( ; (i+2UL) <= M; i+=2UL ) {
1272  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1273  {
1274  auto element( ( IsUpper_v<MT4> )
1275  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1276  :( B.begin(j) ) );
1277  const auto end( ( IsLower_v<MT4> )
1278  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1279  :( B.end(j) ) );
1280 
1281  for( ; element!=end; ++element ) {
1282  C(i ,j) -= A(i ,element->index()) * element->value();
1283  C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1284  }
1285  }
1286  }
1287 
1288  for( ; i<M; ++i ) {
1289  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1290  {
1291  auto element( ( IsUpper_v<MT4> )
1292  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1293  :( B.begin(j) ) );
1294  const auto end( ( IsLower_v<MT4> )
1295  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1296  :( B.end(j) ) );
1297 
1298  for( ; element!=end; ++element )
1299  C(i,j) -= A(i,element->index()) * element->value();
1300  }
1301  }
1302  }
1303  }
1305  //**********************************************************************************************
1306 
1307  //**Optimized subtraction assignment to dense matrices******************************************
1321  template< typename MT3 // Type of the left-hand side target matrix
1322  , typename MT4 // Type of the left-hand side matrix operand
1323  , typename MT5 > // Type of the right-hand side matrix operand
1324  static inline auto selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
1325  -> EnableIf_t< UseOptimizedKernel_v<MT3,MT4,MT5> >
1326  {
1327  const size_t M( A.rows() );
1328  const size_t N( B.columns() );
1329 
1330  BLAZE_INTERNAL_ASSERT( !( LOW || UPP ) || M == N, "Broken invariant detected" );
1331 
1332  {
1333  size_t i( 0UL );
1334 
1335  for( ; (i+4UL) <= M; i+=4UL ) {
1336  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+4UL : N ); ++j )
1337  {
1338  auto element( ( IsUpper_v<MT4> )
1339  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1340  :( B.begin(j) ) );
1341  const auto end( ( IsLower_v<MT4> )
1342  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1343  :( B.end(j) ) );
1344 
1345  const size_t nonzeros( end - element );
1346  const size_t kpos( nonzeros & size_t(-4) );
1347  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1348 
1349  for( size_t k=0UL; k<kpos; k+=4UL )
1350  {
1351  const size_t j1( element->index() );
1352  const ET2 v1( element->value() );
1353  ++element;
1354  const size_t j2( element->index() );
1355  const ET2 v2( element->value() );
1356  ++element;
1357  const size_t j3( element->index() );
1358  const ET2 v3( element->value() );
1359  ++element;
1360  const size_t j4( element->index() );
1361  const ET2 v4( element->value() );
1362  ++element;
1363 
1364  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1365 
1366  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1367  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1368  C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1369  C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1370  }
1371 
1372  for( ; element!=end; ++element )
1373  {
1374  const size_t j1( element->index() );
1375  const ET2 v1( element->value() );
1376 
1377  C(i ,j) -= A(i ,j1) * v1;
1378  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1379  C(i+2UL,j) -= A(i+2UL,j1) * v1;
1380  C(i+3UL,j) -= A(i+3UL,j1) * v1;
1381  }
1382  }
1383  }
1384 
1385  for( ; (i+2UL) <= M; i+=2UL ) {
1386  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+2UL : N ); ++j )
1387  {
1388  auto element( ( IsUpper_v<MT4> )
1389  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1390  :( B.begin(j) ) );
1391  const auto end( ( IsLower_v<MT4> )
1392  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1393  :( B.end(j) ) );
1394 
1395  const size_t nonzeros( end - element );
1396  const size_t kpos( nonzeros & size_t(-4) );
1397  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1398 
1399  for( size_t k=0UL; k<kpos; k+=4UL )
1400  {
1401  const size_t j1( element->index() );
1402  const ET2 v1( element->value() );
1403  ++element;
1404  const size_t j2( element->index() );
1405  const ET2 v2( element->value() );
1406  ++element;
1407  const size_t j3( element->index() );
1408  const ET2 v3( element->value() );
1409  ++element;
1410  const size_t j4( element->index() );
1411  const ET2 v4( element->value() );
1412  ++element;
1413 
1414  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1415 
1416  C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1417  C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1418  }
1419 
1420  for( ; element!=end; ++element )
1421  {
1422  const size_t j1( element->index() );
1423  const ET2 v1( element->value() );
1424 
1425  C(i ,j) -= A(i ,j1) * v1;
1426  C(i+1UL,j) -= A(i+1UL,j1) * v1;
1427  }
1428  }
1429  }
1430 
1431  for( ; i<M; ++i ) {
1432  for( size_t j=( UPP ? i : 0UL ); j<( LOW ? i+1UL : N ); ++j )
1433  {
1434  auto element( ( IsUpper_v<MT4> )
1435  ?( IsStrictlyUpper_v<MT4> ? B.upperBound(i,j) : B.lowerBound(i,j) )
1436  :( B.begin(j) ) );
1437  const auto end( ( IsLower_v<MT4> )
1438  ?( IsStrictlyLower_v<MT4> ? B.lowerBound(i,j) : B.upperBound(i,j) )
1439  :( B.end(j) ) );
1440 
1441  const size_t nonzeros( end - element );
1442  const size_t kpos( nonzeros & size_t(-4) );
1443  BLAZE_INTERNAL_ASSERT( ( nonzeros - ( nonzeros % 4UL ) ) == kpos, "Invalid end calculation" );
1444 
1445  for( size_t k=0UL; k<kpos; k+=4UL )
1446  {
1447  const size_t j1( element->index() );
1448  const ET2 v1( element->value() );
1449  ++element;
1450  const size_t j2( element->index() );
1451  const ET2 v2( element->value() );
1452  ++element;
1453  const size_t j3( element->index() );
1454  const ET2 v3( element->value() );
1455  ++element;
1456  const size_t j4( element->index() );
1457  const ET2 v4( element->value() );
1458  ++element;
1459 
1460  BLAZE_INTERNAL_ASSERT( j1 < j2 && j2 < j3 && j3 < j4, "Invalid sparse matrix index detected" );
1461 
1462  C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1463  }
1464 
1465  for( ; element!=end; ++element )
1466  {
1467  const size_t j1( element->index() );
1468  const ET2 v1( element->value() );
1469 
1470  C(i,j) -= A(i,j1) * v1;
1471  }
1472  }
1473  }
1474  }
1475  }
1477  //**********************************************************************************************
1478 
1479  //**Restructuring subtraction assignment********************************************************
1494  template< typename MT // Type of the target matrix
1495  , bool SO > // Storage order of the target matrix
1496  friend inline auto subAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1497  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1498  {
1500 
1502 
1503  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1504  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1505 
1506  const ForwardFunctor fwd;
1507 
1508  subAssign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1509  }
1511  //**********************************************************************************************
1512 
1513  //**Subtraction assignment to sparse matrices***************************************************
1514  // No special implementation for the subtraction assignment to sparse matrices.
1515  //**********************************************************************************************
1516 
1517  //**Schur product assignment to dense matrices**************************************************
1530  template< typename MT // Type of the target dense matrix
1531  , bool SO > // Storage order of the target dense matrix
1532  friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1533  {
1535 
1539 
1540  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1541  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1542 
1543  const ResultType tmp( serial( rhs ) );
1544  schurAssign( ~lhs, tmp );
1545  }
1547  //**********************************************************************************************
1548 
1549  //**Schur product assignment to sparse matrices*************************************************
1550  // No special implementation for the Schur product assignment to sparse matrices.
1551  //**********************************************************************************************
1552 
1553  //**Multiplication assignment to dense matrices*************************************************
1554  // No special implementation for the multiplication assignment to dense matrices.
1555  //**********************************************************************************************
1556 
1557  //**Multiplication assignment to sparse matrices************************************************
1558  // No special implementation for the multiplication assignment to sparse matrices.
1559  //**********************************************************************************************
1560 
1561  //**SMP assignment to dense matrices************************************************************
1576  template< typename MT // Type of the target dense matrix
1577  , bool SO > // Storage order of the target dense matrix
1578  friend inline auto smpAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1579  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1580  {
1582 
1583  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1584  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1585 
1586  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1587  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1588 
1589  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1590  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1591  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
1592 
1593  smpAssign( ~lhs, A * B );
1594  }
1596  //**********************************************************************************************
1597 
1598  //**SMP assignment to sparse matrices***********************************************************
1613  template< typename MT // Type of the target sparse matrix
1614  , bool SO > // Storage order of the target sparse matrix
1615  friend inline auto smpAssign( SparseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1616  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1617  {
1619 
1620  using TmpType = If_t< SO, OppositeType, ResultType >;
1621 
1628 
1629  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1630  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1631 
1632  const ForwardFunctor fwd;
1633 
1634  const TmpType tmp( rhs );
1635  smpAssign( ~lhs, fwd( tmp ) );
1636  }
1638  //**********************************************************************************************
1639 
1640  //**Restructuring SMP assignment****************************************************************
1655  template< typename MT // Type of the target matrix
1656  , bool SO > // Storage order of the target matrix
1657  friend inline auto smpAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1658  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1659  {
1661 
1663 
1664  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1665  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1666 
1667  const ForwardFunctor fwd;
1668 
1669  smpAssign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1670  }
1672  //**********************************************************************************************
1673 
1674  //**SMP addition assignment to dense matrices***************************************************
1690  template< typename MT // Type of the target dense matrix
1691  , bool SO > // Storage order of the target dense matrix
1692  friend inline auto smpAddAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1693  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1694  {
1696 
1697  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1698  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1699 
1700  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1701  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1702 
1703  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1704  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1705  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
1706 
1707  smpAddAssign( ~lhs, A * B );
1708  }
1710  //**********************************************************************************************
1711 
1712  //**Restructuring SMP addition assignment*******************************************************
1727  template< typename MT // Type of the target matrix
1728  , bool SO > // Storage order of the target matrix
1729  friend inline auto smpAddAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1730  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1731  {
1733 
1735 
1736  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1737  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1738 
1739  const ForwardFunctor fwd;
1740 
1741  smpAddAssign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1742  }
1744  //**********************************************************************************************
1745 
1746  //**SMP addition assignment to sparse matrices**************************************************
1747  // No special implementation for the SMP addition assignment to sparse matrices.
1748  //**********************************************************************************************
1749 
1750  //**SMP subtraction assignment to dense matrices************************************************
1766  template< typename MT // Type of the target dense matrix
1767  , bool SO > // Storage order of the target dense matrix
1768  friend inline auto smpSubAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1769  -> EnableIf_t< IsEvaluationRequired_v<MT,MT1,MT2> >
1770  {
1772 
1773  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1774  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1775 
1776  LT A( rhs.lhs_ ); // Evaluation of the left-hand side dense matrix operand
1777  RT B( rhs.rhs_ ); // Evaluation of the right-hand side sparse matrix operand
1778 
1779  BLAZE_INTERNAL_ASSERT( A.rows() == (~lhs).rows() , "Invalid number of rows" );
1780  BLAZE_INTERNAL_ASSERT( A.columns() == B.rows() , "Invalid matrix sizes" );
1781  BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns(), "Invalid number of columns" );
1782 
1783  smpSubAssign( ~lhs, A * B );
1784  }
1786  //**********************************************************************************************
1787 
1788  //**Restructuring SMP subtraction assignment****************************************************
1803  template< typename MT // Type of the target matrix
1804  , bool SO > // Storage order of the target matrix
1805  friend inline auto smpSubAssign( Matrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1806  -> EnableIf_t< CanExploitSymmetry_v<MT,MT1,MT2> >
1807  {
1809 
1811 
1812  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1813  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1814 
1815  const ForwardFunctor fwd;
1816 
1817  smpSubAssign( ~lhs, fwd( trans( rhs.lhs_ ) * rhs.rhs_ ) );
1818  }
1820  //**********************************************************************************************
1821 
1822  //**SMP subtraction assignment to sparse matrices***********************************************
1823  // No special implementation for the SMP subtraction assignment to sparse matrices.
1824  //**********************************************************************************************
1825 
1826  //**SMP Schur product assignment to dense matrices**********************************************
1839  template< typename MT // Type of the target dense matrix
1840  , bool SO > // Storage order of the target dense matrix
1841  friend inline void smpSchurAssign( DenseMatrix<MT,SO>& lhs, const DMatTSMatMultExpr& rhs )
1842  {
1844 
1848 
1849  BLAZE_INTERNAL_ASSERT( (~lhs).rows() == rhs.rows() , "Invalid number of rows" );
1850  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
1851 
1852  const ResultType tmp( rhs );
1853  smpSchurAssign( ~lhs, tmp );
1854  }
1856  //**********************************************************************************************
1857 
1858  //**SMP Schur product assignment to sparse matrices*********************************************
1859  // No special implementation for the SMP Schur product assignment to sparse matrices.
1860  //**********************************************************************************************
1861 
1862  //**SMP multiplication assignment to dense matrices*********************************************
1863  // No special implementation for the SMP multiplication assignment to dense matrices.
1864  //**********************************************************************************************
1865 
1866  //**SMP multiplication assignment to sparse matrices********************************************
1867  // No special implementation for the SMP multiplication assignment to sparse matrices.
1868  //**********************************************************************************************
1869 
1870  //**Compile time checks*************************************************************************
1879  //**********************************************************************************************
1880 };
1881 //*************************************************************************************************
1882 
1883 
1884 
1885 
1886 //=================================================================================================
1887 //
1888 // GLOBAL BINARY ARITHMETIC OPERATORS
1889 //
1890 //=================================================================================================
1891 
1892 //*************************************************************************************************
1905 template< typename MT1 // Type of the left-hand side dense matrix
1906  , typename MT2 // Type of the right-hand side sparse matrix
1907  , DisableIf_t< ( IsIdentity_v<MT2> &&
1908  IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > ) ||
1909  IsZero_v<MT2> >* = nullptr >
1910 inline const DMatTSMatMultExpr<MT1,MT2,false,false,false,false>
1911  dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1912 {
1914 
1915  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
1916 
1917  return DMatTSMatMultExpr<MT1,MT2,false,false,false,false>( ~lhs, ~rhs );
1918 }
1920 //*************************************************************************************************
1921 
1922 
1923 //*************************************************************************************************
1937 template< typename MT1 // Type of the left-hand side dense matrix
1938  , typename MT2 // Type of the right-hand side sparse matrix
1939  , EnableIf_t< IsIdentity_v<MT2> &&
1940  IsSame_v< ElementType_t<MT1>, ElementType_t<MT2> > >* = nullptr >
1941 inline const MT1&
1942  dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1943 {
1945 
1946  MAYBE_UNUSED( rhs );
1947 
1948  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
1949 
1950  return (~lhs);
1951 }
1953 //*************************************************************************************************
1954 
1955 
1956 //*************************************************************************************************
1969 template< typename MT1 // Type of the left-hand side dense matrix
1970  , typename MT2 // Type of the right-hand side sparse matrix
1971  , EnableIf_t< IsZero_v<MT2> >* = nullptr >
1972 inline decltype(auto)
1973  dmattsmatmult( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
1974 {
1976 
1977  BLAZE_INTERNAL_ASSERT( (~lhs).columns() == (~rhs).rows(), "Invalid matrix sizes" );
1978 
1979  using ReturnType = const MultTrait_t< ResultType_t<MT1>, ResultType_t<MT2> >;
1980 
1982  BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE( ReturnType );
1983 
1984  return ReturnType( (~lhs).rows(), (~rhs).columns() );
1985 }
1987 //*************************************************************************************************
1988 
1989 
1990 //*************************************************************************************************
2020 template< typename MT1 // Type of the left-hand side dense matrix
2021  , typename MT2 > // Type of the right-hand side sparse matrix
2022 inline decltype(auto)
2023  operator*( const DenseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,true>& rhs )
2024 {
2026 
2027  if( (~lhs).columns() != (~rhs).rows() ) {
2028  BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
2029  }
2030 
2031  return dmattsmatmult( ~lhs, ~rhs );
2032 }
2033 //*************************************************************************************************
2034 
2035 
2036 
2037 
2038 //=================================================================================================
2039 //
2040 // GLOBAL FUNCTIONS
2041 //
2042 //=================================================================================================
2043 
2044 //*************************************************************************************************
2068 template< typename MT1 // Type of the left-hand side dense matrix
2069  , typename MT2 // Type of the right-hand side sparse matrix
2070  , bool SF // Symmetry flag
2071  , bool HF // Hermitian flag
2072  , bool LF // Lower flag
2073  , bool UF > // Upper flag
2074 inline decltype(auto) declsym( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2075 {
2077 
2078  if( !isSquare( dm ) ) {
2079  BLAZE_THROW_INVALID_ARGUMENT( "Invalid symmetric matrix specification" );
2080  }
2081 
2082  using ReturnType = const DMatTSMatMultExpr<MT1,MT2,true,HF,LF,UF>;
2083  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2084 }
2086 //*************************************************************************************************
2087 
2088 
2089 //*************************************************************************************************
2113 template< typename MT1 // Type of the left-hand side dense matrix
2114  , typename MT2 // Type of the right-hand side sparse matrix
2115  , bool SF // Symmetry flag
2116  , bool HF // Hermitian flag
2117  , bool LF // Lower flag
2118  , bool UF > // Upper flag
2119 inline decltype(auto) declherm( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2120 {
2122 
2123  if( !isSquare( dm ) ) {
2124  BLAZE_THROW_INVALID_ARGUMENT( "Invalid Hermitian matrix specification" );
2125  }
2126 
2127  using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,true,LF,UF>;
2128  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2129 }
2131 //*************************************************************************************************
2132 
2133 
2134 //*************************************************************************************************
2158 template< typename MT1 // Type of the left-hand side dense matrix
2159  , typename MT2 // Type of the right-hand side sparse matrix
2160  , bool SF // Symmetry flag
2161  , bool HF // Hermitian flag
2162  , bool LF // Lower flag
2163  , bool UF > // Upper flag
2164 inline decltype(auto) decllow( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2165 {
2167 
2168  if( !isSquare( dm ) ) {
2169  BLAZE_THROW_INVALID_ARGUMENT( "Invalid lower matrix specification" );
2170  }
2171 
2172  using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,true,UF>;
2173  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2174 }
2176 //*************************************************************************************************
2177 
2178 
2179 //*************************************************************************************************
2203 template< typename MT1 // Type of the left-hand side dense matrix
2204  , typename MT2 // Type of the right-hand side sparse matrix
2205  , bool SF // Symmetry flag
2206  , bool HF // Hermitian flag
2207  , bool LF // Lower flag
2208  , bool UF > // Upper flag
2209 inline decltype(auto) declupp( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2210 {
2212 
2213  if( !isSquare( dm ) ) {
2214  BLAZE_THROW_INVALID_ARGUMENT( "Invalid upper matrix specification" );
2215  }
2216 
2217  using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,true>;
2218  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2219 }
2221 //*************************************************************************************************
2222 
2223 
2224 //*************************************************************************************************
2248 template< typename MT1 // Type of the left-hand side dense matrix
2249  , typename MT2 // Type of the right-hand side sparse matrix
2250  , bool SF // Symmetry flag
2251  , bool HF // Hermitian flag
2252  , bool LF // Lower flag
2253  , bool UF > // Upper flag
2254 inline decltype(auto) decldiag( const DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
2255 {
2257 
2258  if( !isSquare( dm ) ) {
2259  BLAZE_THROW_INVALID_ARGUMENT( "Invalid diagonal matrix specification" );
2260  }
2261 
2262  using ReturnType = const DMatTSMatMultExpr<MT1,MT2,SF,HF,true,true>;
2263  return ReturnType( dm.leftOperand(), dm.rightOperand() );
2264 }
2266 //*************************************************************************************************
2267 
2268 
2269 
2270 
2271 //=================================================================================================
2272 //
2273 // SIZE SPECIALIZATIONS
2274 //
2275 //=================================================================================================
2276 
2277 //*************************************************************************************************
2279 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2280 struct Size< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 0UL >
2281  : public Size<MT1,0UL>
2282 {};
2283 
2284 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2285 struct Size< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF>, 1UL >
2286  : public Size<MT2,1UL>
2287 {};
2289 //*************************************************************************************************
2290 
2291 
2292 
2293 
2294 //=================================================================================================
2295 //
2296 // ISALIGNED SPECIALIZATIONS
2297 //
2298 //=================================================================================================
2299 
2300 //*************************************************************************************************
2302 template< typename MT1, typename MT2, bool SF, bool HF, bool LF, bool UF >
2303 struct IsAligned< DMatTSMatMultExpr<MT1,MT2,SF,HF,LF,UF> >
2304  : public IsAligned<MT1>
2305 {};
2307 //*************************************************************************************************
2308 
2309 } // namespace blaze
2310 
2311 #endif
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition: Column.h:133
Headerfile for the generic min algorithm.
Header file for the blaze::checked and blaze::unchecked instances.
Header file for the decldiag trait.
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition: DMatDeclDiagExpr.h:975
Header file for basic type definitions.
typename If< Condition, T1, T2 >::Type If_t
Auxiliary alias template for the If class template.The If_t alias template provides a convenient shor...
Definition: If.h:109
Header file for the declherm trait.
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.The ResultType_t alias declaration provides ...
Definition: Aliases.h:390
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: DMatTSMatMultExpr.h:359
Header file for the serial shim.
Header file for the IsDiagonal type trait.
Base template for the DeclUppTrait class.
Definition: DeclUppTrait.h:134
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: DMatTSMatMultExpr.h:251
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type,...
Definition: DenseMatrix.h:61
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:134
Header file for the DeclUpp functor.
Header file for the IsSame and IsStrictlySame type traits.
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:372
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:595
DMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the DMatTSMatMultExpr class.
Definition: DMatTSMatMultExpr.h:269
LeftOperand leftOperand() const noexcept
Returns the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:369
constexpr Unchecked unchecked
Global Unchecked instance.The blaze::unchecked instance is an optional token for the creation of view...
Definition: Check.h:138
Constraint on the data type.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: DMatTSMatMultExpr.h:333
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:154
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:142
Header file for the MAYBE_UNUSED function template.
Header file for the IsIdentity type trait.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: DMatTSMatMultExpr.h:423
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition: DMatDeclUppExpr.h:1001
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:233
ResultType_t< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:132
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Header file for the reset shim.
Constraints on the storage order of matrix types.
ElementType_t< ResultType > ElementType
Resulting element type.
Definition: DMatTSMatMultExpr.h:237
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
constexpr size_t columns(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of columns of the matrix.
Definition: Matrix.h:514
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes....
Definition: DenseMatrix.h:81
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes....
Definition: Forward.h:145
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.The ElementType_t alias declaration provide...
Definition: Aliases.h:170
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:236
ResultType_t< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:133
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: DMatTSMatMultExpr.h:235
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition: DMatTSMatMultExpr.h:256
Constraint on the data type.
Constraint on the data type.
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:431
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:248
Headerfile for the generic max algorithm.
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:58
Header file for the DeclLow functor.
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: DMatTSMatMultExpr.h:403
Header file for the If class template.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: DMatTSMatMultExpr.h:391
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is not a zero vector or matrix type,...
Definition: Zero.h:61
Generic wrapper for the decllow() function.
Definition: DeclLow.h:59
static constexpr bool HERM
Flag for Hermitian matrices.
Definition: DMatTSMatMultExpr.h:152
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1162
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition: DMatTSMatMultExpr.h:259
Header file for the decllow trait.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the DenseMatrix base class.
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: DMatTSMatMultExpr.h:379
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition: DMatDeclLowExpr.h:1001
CompositeType_t< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:136
Header file for the IsLower type trait.
constexpr void MAYBE_UNUSED(const Args &...)
Suppression of unused parameter warnings.
Definition: MaybeUnused.h:81
Header file for the IsAligned type trait.
static constexpr bool SYM
Flag for symmetric matrices.
Definition: DMatTSMatMultExpr.h:151
Generic wrapper for the null function.
Definition: Noop.h:60
Header file for the IsTriangular type trait.
Base template for the DeclSymTrait class.
Definition: DeclSymTrait.h:134
Constraints on the storage order of matrix types.
Header file for the exception macros of the math module.
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition: DMatDMatMapExpr.h:1198
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: DMatTSMatMultExpr.h:349
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:438
Header file for the DeclDiag functor.
ElementType_t< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:135
Constraint on the data type.
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: DMatTSMatMultExpr.h:413
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:103
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.The OppositeType_t alias declaration provi...
Definition: Aliases.h:270
Header file for the conjugate shim.
Header file for the declupp trait.
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:245
Base class for all matrix/matrix multiplication expression templates.The MatMatMultExpr class serves ...
Definition: MatMatMultExpr.h:67
static constexpr bool LOW
Flag for lower matrices.
Definition: DMatTSMatMultExpr.h:153
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type,...
Definition: Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.The TransposeType_t alias declaration pro...
Definition: Aliases.h:470
Header file for run time assertion macros.
Base template for the DeclHermTrait class.
Definition: DeclHermTrait.h:134
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.The CompositeType_t alias declaration pro...
Definition: Aliases.h:90
Base template for the MultTrait class.
Definition: MultTrait.h:146
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:131
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition: Row.h:133
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:430
Header file for the IsZero type trait.
Header file for the declsym trait.
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:94
Header file for all forward declarations for expression class templates.
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition: DMatDeclSymExpr.h:1002
Expression object for dense matrix-transpose sparse matrix multiplications.The DMatTSMatMultExpr clas...
Definition: DMatTSMatMultExpr.h:126
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:100
Constraint on the data type.
Constraints on the storage order of matrix types.
Generic wrapper for the declherm() function.
Definition: DeclHerm.h:59
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:808
Header file for the Noop functor.
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.In case the given data type T requires an intermediate evaluation within ...
Definition: RequiresEvaluation.h:81
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:194
constexpr size_t rows(const Matrix< MT, SO > &matrix) noexcept
Returns the current number of rows of the matrix.
Definition: Matrix.h:498
CompositeType_t< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:137
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
const ElementType ReturnType
Return type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:238
Generic wrapper for the declupp() function.
Definition: DeclUpp.h:59
decltype(auto) trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:765
const ResultType CompositeType
Data type for composite expression templates.
Definition: DMatTSMatMultExpr.h:239
Base template for the DeclLowTrait class.
Definition: DeclLowTrait.h:134
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition: DMatDeclHermExpr.h:1002
static constexpr bool UPP
Flag for upper matrices.
Definition: DMatTSMatMultExpr.h:154
Header file for the IsComputation type trait class.
Header file for the IsBuiltin type trait.
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:162
Base class for all compute expression templates.The Computation class serves as a tag for all computa...
Definition: Computation.h:66
Header file for the IntegralConstant class template.
Generic wrapper for the decldiag() function.
Definition: DeclDiag.h:59
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:242
Header file for the DeclHerm functor.
Header file for the IsUpper type trait.
typename DisableIf< Condition, T >::Type DisableIf_t
Auxiliary type for the DisableIf class template.The DisableIf_t alias declaration provides a convenie...
Definition: DisableIf.h:138
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition: DMatMapExpr.h:1324
Constraint on the data type.
Generic wrapper for the declsym() function.
Definition: DeclSym.h:59
Base template for the DeclDiagTrait class.
Definition: DeclDiagTrait.h:134
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition: Matrix.h:951
Header file for the IsResizable type trait.
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:147
Header file for the Size type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_ZERO_TYPE(T)
Constraint on the data type.In case the given data type T is a zero vector or matrix type,...
Definition: Zero.h:81
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression,...
Definition: Assert.h:101
Header file for the DeclSym functor.
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type,...
Definition: SparseMatrix.h:61
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: DMatTSMatMultExpr.h:284
Header file for the IsExpression type trait class.
Header file for the function trace functionality.