35 #ifndef _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
36 #define _BLAZE_MATH_EXPRESSIONS_DMATTSMATMULTEXPR_H_
125 template<
typename MT1
158 template<
typename T1,
typename T2,
typename T3 >
159 struct CanExploitSymmetry {
160 enum :
bool { value = IsSymmetric<T2>::value };
171 template<
typename T1,
typename T2,
typename T3 >
172 struct IsEvaluationRequired {
173 enum :
bool { value = ( evaluateLeft || evaluateRight ) &&
174 !CanExploitSymmetry<T1,T2,T3>::value };
184 template<
typename T1,
typename T2,
typename T3 >
185 struct UseOptimizedKernel {
187 !IsDiagonal<T2>::value &&
188 !IsResizable< ElementType_<T1> >::value &&
189 !IsResizable<ET2>::value };
219 enum :
bool { simdEnabled =
false };
222 enum :
bool { smpAssignable = !evaluateLeft && MT1::smpAssignable &&
223 !evaluateRight && MT2::smpAssignable };
273 :(
lhs_.columns() ) ) );
277 const size_t n(
end - begin );
295 inline ReturnType
at(
size_t i,
size_t j )
const {
296 if( i >=
lhs_.rows() ) {
299 if( j >=
rhs_.columns() ) {
311 inline size_t rows() const noexcept {
322 return rhs_.columns();
352 template<
typename T >
353 inline bool canAlias(
const T* alias )
const noexcept {
354 return (
lhs_.isAliased( alias ) ||
rhs_.isAliased( alias ) );
364 template<
typename T >
365 inline bool isAliased(
const T* alias )
const noexcept {
366 return (
lhs_.isAliased( alias ) ||
rhs_.isAliased( alias ) );
376 return lhs_.isAligned();
386 return (
rows() *
columns() >= SMP_DMATTSMATMULT_THRESHOLD );
409 template<
typename MT
426 DMatTSMatMultExpr::selectAssignKernel( ~lhs, A, B );
445 template<
typename MT3
449 selectAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
455 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
457 const size_t jend(
min( jj+block, B.columns() ) );
461 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
462 for(
size_t j=jj; j<jend; ++j )
471 if( element ==
end ) {
479 C(i ,j) = A(i ,element->index()) * element->value();
480 C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
481 C(i+2UL,j) = A(i+2UL,element->index()) * element->value();
482 C(i+3UL,j) = A(i+3UL,element->index()) * element->value();
484 for( ; element!=
end; ++element ) {
485 C(i ,j) += A(i ,element->index()) * element->value();
486 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
487 C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
488 C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
493 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
494 for(
size_t j=jj; j<jend; ++j )
496 ConstIterator element( ( IsUpper<MT4>::value )
497 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
499 const ConstIterator
end( ( IsLower<MT4>::value )
500 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
503 if( element ==
end ) {
509 C(i ,j) = A(i ,element->index()) * element->value();
510 C(i+1UL,j) = A(i+1UL,element->index()) * element->value();
512 for( ; element!=
end; ++element ) {
513 C(i ,j) += A(i ,element->index()) * element->value();
514 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
519 for( ; i<A.rows(); ++i ) {
520 for(
size_t j=jj; j<jend; ++j )
522 ConstIterator element( ( IsUpper<MT4>::value )
523 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
525 const ConstIterator
end( ( IsLower<MT4>::value )
526 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
529 if( element ==
end ) {
534 C(i,j) = A(i,element->index()) * element->value();
536 for( ; element!=
end; ++element )
537 C(i,j) += A(i,element->index()) * element->value();
559 template<
typename MT3
562 static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
563 selectAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
567 const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 256UL );
571 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
573 const size_t jend(
min( jj+block, B.columns() ) );
577 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
578 for(
size_t j=jj; j<jend; ++j )
580 ConstIterator element( ( IsUpper<MT4>::value )
581 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
583 const ConstIterator
end( ( IsLower<MT4>::value )
584 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
587 const size_t nonzeros(
end - element );
588 const size_t kpos( nonzeros &
size_t(-4) );
591 for(
size_t k=0UL; k<kpos; k+=4UL )
593 const size_t j1( element->index() );
594 const ET2 v1( element->value() );
596 const size_t j2( element->index() );
597 const ET2 v2( element->value() );
599 const size_t j3( element->index() );
600 const ET2 v3( element->value() );
602 const size_t j4( element->index() );
603 const ET2 v4( element->value() );
608 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
609 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
610 C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
611 C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
614 for( ; element!=
end; ++element )
616 const size_t j1( element->index() );
617 const ET2 v1( element->value() );
619 C(i ,j) += A(i ,j1) * v1;
620 C(i+1UL,j) += A(i+1UL,j1) * v1;
621 C(i+2UL,j) += A(i+2UL,j1) * v1;
622 C(i+3UL,j) += A(i+3UL,j1) * v1;
627 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
628 for(
size_t j=jj; j<jend; ++j )
630 ConstIterator element( ( IsUpper<MT4>::value )
631 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
633 const ConstIterator
end( ( IsLower<MT4>::value )
634 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
637 const size_t nonzeros(
end - element );
638 const size_t kpos( nonzeros &
size_t(-4) );
641 for(
size_t k=0UL; k<kpos; k+=4UL )
643 const size_t j1( element->index() );
644 const ET2 v1( element->value() );
646 const size_t j2( element->index() );
647 const ET2 v2( element->value() );
649 const size_t j3( element->index() );
650 const ET2 v3( element->value() );
652 const size_t j4( element->index() );
653 const ET2 v4( element->value() );
658 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
659 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
662 for( ; element!=
end; ++element )
664 const size_t j1( element->index() );
665 const ET2 v1( element->value() );
667 C(i ,j) += A(i ,j1) * v1;
668 C(i+1UL,j) += A(i+1UL,j1) * v1;
673 for( ; i<A.rows(); ++i ) {
674 for(
size_t j=jj; j<jend; ++j )
676 ConstIterator element( ( IsUpper<MT4>::value )
677 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
679 const ConstIterator
end( ( IsLower<MT4>::value )
680 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
683 const size_t nonzeros(
end - element );
684 const size_t kpos( nonzeros &
size_t(-4) );
687 for(
size_t k=0UL; k<kpos; k+=4UL )
689 const size_t j1( element->index() );
690 const ET2 v1( element->value() );
692 const size_t j2( element->index() );
693 const ET2 v2( element->value() );
695 const size_t j3( element->index() );
696 const ET2 v3( element->value() );
698 const size_t j4( element->index() );
699 const ET2 v4( element->value() );
704 C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
707 for( ; element!=
end; ++element )
709 const size_t j1( element->index() );
710 const ET2 v1( element->value() );
712 C(i,j) += A(i,j1) * v1;
734 template<
typename MT
736 friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
741 typedef IfTrue_< SO, OppositeType, ResultType > TmpType;
753 const TmpType tmp(
serial( rhs ) );
774 template<
typename MT
776 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
784 assign( ~lhs,
trans( rhs.lhs_ ) * rhs.rhs_ );
802 template<
typename MT
804 friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
812 LT A(
serial( rhs.lhs_ ) );
813 RT B(
serial( rhs.rhs_ ) );
819 DMatTSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
838 template<
typename MT3
841 static inline DisableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
842 selectAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
846 const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 256UL );
848 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
850 const size_t jend(
min( jj+block, B.columns() ) );
854 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
855 for(
size_t j=jj; j<jend; ++j )
857 ConstIterator element( ( IsUpper<MT4>::value )
858 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
860 const ConstIterator
end( ( IsLower<MT4>::value )
861 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
864 for( ; element!=
end; ++element ) {
865 C(i ,j) += A(i ,element->index()) * element->value();
866 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
867 C(i+2UL,j) += A(i+2UL,element->index()) * element->value();
868 C(i+3UL,j) += A(i+3UL,element->index()) * element->value();
873 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
874 for(
size_t j=jj; j<jend; ++j )
876 ConstIterator element( ( IsUpper<MT4>::value )
877 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
879 const ConstIterator
end( ( IsLower<MT4>::value )
880 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
883 for( ; element!=
end; ++element ) {
884 C(i ,j) += A(i ,element->index()) * element->value();
885 C(i+1UL,j) += A(i+1UL,element->index()) * element->value();
890 for( ; i<A.rows(); ++i ) {
891 for(
size_t j=jj; j<jend; ++j )
893 ConstIterator element( ( IsUpper<MT4>::value )
894 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
896 const ConstIterator
end( ( IsLower<MT4>::value )
897 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
900 for( ; element!=
end; ++element )
901 C(i,j) += A(i,element->index()) * element->value();
923 template<
typename MT3
926 static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
927 selectAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
931 const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 256UL );
933 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
935 const size_t jend(
min( jj+block, B.columns() ) );
939 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
940 for(
size_t j=jj; j<jend; ++j )
942 ConstIterator element( ( IsUpper<MT4>::value )
943 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
945 const ConstIterator
end( ( IsLower<MT4>::value )
946 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
949 const size_t nonzeros(
end - element );
950 const size_t kpos( nonzeros &
size_t(-4) );
953 for(
size_t k=0UL; k<kpos; k+=4UL )
955 const size_t j1( element->index() );
956 const ET2 v1( element->value() );
958 const size_t j2( element->index() );
959 const ET2 v2( element->value() );
961 const size_t j3( element->index() );
962 const ET2 v3( element->value() );
964 const size_t j4( element->index() );
965 const ET2 v4( element->value() );
970 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
971 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
972 C(i+2UL,j) += A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
973 C(i+3UL,j) += A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
976 for( ; element!=
end; ++element )
978 const size_t j1( element->index() );
979 const ET2 v1( element->value() );
981 C(i ,j) += A(i ,j1) * v1;
982 C(i+1UL,j) += A(i+1UL,j1) * v1;
983 C(i+2UL,j) += A(i+2UL,j1) * v1;
984 C(i+3UL,j) += A(i+3UL,j1) * v1;
989 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
990 for(
size_t j=jj; j<jend; ++j )
992 ConstIterator element( ( IsUpper<MT4>::value )
993 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
995 const ConstIterator
end( ( IsLower<MT4>::value )
996 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
999 const size_t nonzeros(
end - element );
1000 const size_t kpos( nonzeros &
size_t(-4) );
1003 for(
size_t k=0UL; k<kpos; k+=4UL )
1005 const size_t j1( element->index() );
1006 const ET2 v1( element->value() );
1008 const size_t j2( element->index() );
1009 const ET2 v2( element->value() );
1011 const size_t j3( element->index() );
1012 const ET2 v3( element->value() );
1014 const size_t j4( element->index() );
1015 const ET2 v4( element->value() );
1020 C(i ,j) += A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1021 C(i+1UL,j) += A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1024 for( ; element!=
end; ++element )
1026 const size_t j1( element->index() );
1027 const ET2 v1( element->value() );
1029 C(i ,j) += A(i ,j1) * v1;
1030 C(i+1UL,j) += A(i+1UL,j1) * v1;
1035 for( ; i<A.rows(); ++i ) {
1036 for(
size_t j=jj; j<jend; ++j )
1038 ConstIterator element( ( IsUpper<MT4>::value )
1039 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1041 const ConstIterator
end( ( IsLower<MT4>::value )
1042 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
1045 const size_t nonzeros(
end - element );
1046 const size_t kpos( nonzeros &
size_t(-4) );
1049 for(
size_t k=0UL; k<kpos; k+=4UL )
1051 const size_t j1( element->index() );
1052 const ET2 v1( element->value() );
1054 const size_t j2( element->index() );
1055 const ET2 v2( element->value() );
1057 const size_t j3( element->index() );
1058 const ET2 v3( element->value() );
1060 const size_t j4( element->index() );
1061 const ET2 v4( element->value() );
1066 C(i,j) += A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1069 for( ; element!=
end; ++element )
1071 const size_t j1( element->index() );
1072 const ET2 v1( element->value() );
1074 C(i,j) += A(i,j1) * v1;
1098 template<
typename MT
1100 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1110 addAssign( ~lhs,
trans( rhs.lhs_ ) * rhs.rhs_ );
1132 template<
typename MT
1134 friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1142 LT A(
serial( rhs.lhs_ ) );
1143 RT B(
serial( rhs.rhs_ ) );
1149 DMatTSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
1168 template<
typename MT3
1171 static inline DisableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
1172 selectSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
1176 const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 256UL );
1178 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
1180 const size_t jend(
min( jj+block, B.columns() ) );
1184 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
1185 for(
size_t j=jj; j<jend; ++j )
1187 ConstIterator element( ( IsUpper<MT4>::value )
1188 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1190 const ConstIterator
end( ( IsLower<MT4>::value )
1191 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1194 for( ; element!=
end; ++element ) {
1195 C(i ,j) -= A(i ,element->index()) * element->value();
1196 C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1197 C(i+2UL,j) -= A(i+2UL,element->index()) * element->value();
1198 C(i+3UL,j) -= A(i+3UL,element->index()) * element->value();
1203 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
1204 for(
size_t j=jj; j<jend; ++j )
1206 ConstIterator element( ( IsUpper<MT4>::value )
1207 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1209 const ConstIterator
end( ( IsLower<MT4>::value )
1210 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1213 for( ; element!=
end; ++element ) {
1214 C(i ,j) -= A(i ,element->index()) * element->value();
1215 C(i+1UL,j) -= A(i+1UL,element->index()) * element->value();
1220 for( ; i<A.rows(); ++i ) {
1221 for(
size_t j=jj; j<jend; ++j )
1223 ConstIterator element( ( IsUpper<MT4>::value )
1224 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1226 const ConstIterator
end( ( IsLower<MT4>::value )
1227 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
1230 for( ; element!=
end; ++element )
1231 C(i,j) -= A(i,element->index()) * element->value();
1253 template<
typename MT3
1256 static inline EnableIf_< UseOptimizedKernel<MT3,MT4,MT5> >
1257 selectSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
1261 const size_t block( IsRowMajorMatrix<MT3>::value ? B.columns() : 256UL );
1263 for(
size_t jj=0UL; jj<B.columns(); jj+=block )
1265 const size_t jend(
min( jj+block, B.columns() ) );
1269 for( ; (i+4UL) <= A.rows(); i+=4UL ) {
1270 for(
size_t j=jj; j<jend; ++j )
1272 ConstIterator element( ( IsUpper<MT4>::value )
1273 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1275 const ConstIterator
end( ( IsLower<MT4>::value )
1276 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+4UL,j) : B.upperBound(i+4UL,j) )
1279 const size_t nonzeros(
end - element );
1280 const size_t kpos( nonzeros &
size_t(-4) );
1283 for(
size_t k=0UL; k<kpos; k+=4UL )
1285 const size_t j1( element->index() );
1286 const ET2 v1( element->value() );
1288 const size_t j2( element->index() );
1289 const ET2 v2( element->value() );
1291 const size_t j3( element->index() );
1292 const ET2 v3( element->value() );
1294 const size_t j4( element->index() );
1295 const ET2 v4( element->value() );
1300 C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1301 C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1302 C(i+2UL,j) -= A(i+2UL,j1) * v1 + A(i+2UL,j2) * v2 + A(i+2UL,j3) * v3 + A(i+2UL,j4) * v4;
1303 C(i+3UL,j) -= A(i+3UL,j1) * v1 + A(i+3UL,j2) * v2 + A(i+3UL,j3) * v3 + A(i+3UL,j4) * v4;
1306 for( ; element!=
end; ++element )
1308 const size_t j1( element->index() );
1309 const ET2 v1( element->value() );
1311 C(i ,j) -= A(i ,j1) * v1;
1312 C(i+1UL,j) -= A(i+1UL,j1) * v1;
1313 C(i+2UL,j) -= A(i+2UL,j1) * v1;
1314 C(i+3UL,j) -= A(i+3UL,j1) * v1;
1319 for( ; (i+2UL) <= A.rows(); i+=2UL ) {
1320 for(
size_t j=jj; j<jend; ++j )
1322 ConstIterator element( ( IsUpper<MT4>::value )
1323 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1325 const ConstIterator
end( ( IsLower<MT4>::value )
1326 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i+2UL,j) : B.upperBound(i+2UL,j) )
1329 const size_t nonzeros(
end - element );
1330 const size_t kpos( nonzeros &
size_t(-4) );
1333 for(
size_t k=0UL; k<kpos; k+=4UL )
1335 const size_t j1( element->index() );
1336 const ET2 v1( element->value() );
1338 const size_t j2( element->index() );
1339 const ET2 v2( element->value() );
1341 const size_t j3( element->index() );
1342 const ET2 v3( element->value() );
1344 const size_t j4( element->index() );
1345 const ET2 v4( element->value() );
1350 C(i ,j) -= A(i ,j1) * v1 + A(i ,j2) * v2 + A(i ,j3) * v3 + A(i ,j4) * v4;
1351 C(i+1UL,j) -= A(i+1UL,j1) * v1 + A(i+1UL,j2) * v2 + A(i+1UL,j3) * v3 + A(i+1UL,j4) * v4;
1354 for( ; element!=
end; ++element )
1356 const size_t j1( element->index() );
1357 const ET2 v1( element->value() );
1359 C(i ,j) -= A(i ,j1) * v1;
1360 C(i+1UL,j) -= A(i+1UL,j1) * v1;
1365 for( ; i<A.rows(); ++i ) {
1366 for(
size_t j=jj; j<jend; ++j )
1368 ConstIterator element( ( IsUpper<MT4>::value )
1369 ?( IsStrictlyUpper<MT4>::value ? B.upperBound(i,j) : B.lowerBound(i,j) )
1371 const ConstIterator
end( ( IsLower<MT4>::value )
1372 ?( IsStrictlyLower<MT4>::value ? B.lowerBound(i,j) : B.upperBound(i,j) )
1375 const size_t nonzeros(
end - element );
1376 const size_t kpos( nonzeros &
size_t(-4) );
1379 for(
size_t k=0UL; k<kpos; k+=4UL )
1381 const size_t j1( element->index() );
1382 const ET2 v1( element->value() );
1384 const size_t j2( element->index() );
1385 const ET2 v2( element->value() );
1387 const size_t j3( element->index() );
1388 const ET2 v3( element->value() );
1390 const size_t j4( element->index() );
1391 const ET2 v4( element->value() );
1396 C(i,j) -= A(i,j1) * v1 + A(i,j2) * v2 + A(i,j3) * v3 + A(i,j4) * v4;
1399 for( ; element!=
end; ++element )
1401 const size_t j1( element->index() );
1402 const ET2 v1( element->value() );
1404 C(i,j) -= A(i,j1) * v1;
1428 template<
typename MT
1430 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1440 subAssign( ~lhs,
trans( rhs.lhs_ ) * rhs.rhs_ );
1472 template<
typename MT
1474 friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1509 template<
typename MT
1511 friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1516 typedef IfTrue_< SO, OppositeType, ResultType > TmpType;
1528 const TmpType tmp( rhs );
1549 template<
typename MT
1551 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1582 template<
typename MT
1584 friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1619 template<
typename MT
1621 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1656 template<
typename MT
1658 friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
1693 template<
typename MT
1695 friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
1773 template<
typename T1
1775 inline const DMatTSMatMultExpr<T1,T2>
1799 template<
typename MT1,
typename MT2 >
1816 template<
typename MT1,
typename MT2 >
1833 template<
typename MT1,
typename MT2 >
1851 template<
typename MT1,
typename MT2 >
1853 :
public BoolConstant< IsLower<MT1>::value && IsLower<MT2>::value >
1869 template<
typename MT1,
typename MT2 >
1871 :
public BoolConstant< IsUniLower<MT1>::value && IsUniLower<MT2>::value >
1887 template<
typename MT1,
typename MT2 >
1889 :
public BoolConstant< Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
1890 , And< IsStrictlyLower<MT2>, IsLower<MT1> > >::value >
1906 template<
typename MT1,
typename MT2 >
1908 :
public BoolConstant< IsUpper<MT1>::value && IsUpper<MT2>::value >
1924 template<
typename MT1,
typename MT2 >
1926 :
public BoolConstant< IsUniUpper<MT1>::value && IsUniUpper<MT2>::value >
1942 template<
typename MT1,
typename MT2 >
1944 :
public BoolConstant< Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
1945 , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >::value >
1961 template<
typename MT1,
typename MT2,
typename VT >
1966 using Type = If_< And< IsDenseMatrix<MT1>, IsRowMajorMatrix<MT1>
1967 , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2>
1968 , IsDenseVector<VT>, IsColumnVector<VT> >
1969 , DMatDVecMultExprTrait_< MT1, TSMatDVecMultExprTrait_<MT2,VT> >
1979 template<
typename MT1,
typename MT2,
typename VT >
1984 using Type = If_< And< IsDenseMatrix<MT1>, IsRowMajorMatrix<MT1>
1985 , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2>
1986 , IsSparseVector<VT>, IsColumnVector<VT> >
1987 , DMatSVecMultExprTrait_< MT1, TSMatSVecMultExprTrait_<MT2,VT> >
1997 template<
typename VT,
typename MT1,
typename MT2 >
2002 using Type = If_< And< IsDenseVector<VT>, IsRowVector<VT>
2003 , IsDenseMatrix<MT1>, IsRowMajorMatrix<MT1>
2004 , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2> >
2005 , TDVecTSMatMultExprTrait_< TDVecDMatMultExprTrait_<VT,MT1>, MT2 >
2015 template<
typename VT,
typename MT1,
typename MT2 >
2020 using Type = If_< And< IsSparseVector<VT>, IsRowVector<VT>
2021 , IsDenseMatrix<MT1>, IsRowMajorMatrix<MT1>
2022 , IsSparseMatrix<MT2>, IsColumnMajorMatrix<MT2> >
2023 , TDVecTSMatMultExprTrait_< TSVecDMatMultExprTrait_<VT,MT1>, MT2 >
2033 template<
typename MT1,
typename MT2,
bool AF >
2038 using Type = MultExprTrait_< SubmatrixExprTrait_<const MT1,AF>
2039 , SubmatrixExprTrait_<const MT2,AF> >;
2048 template<
typename MT1,
typename MT2 >
2053 using Type = MultExprTrait_< RowExprTrait_<const MT1>, MT2 >;
2062 template<
typename MT1,
typename MT2 >
2067 using Type = MultExprTrait_< MT1, ColumnExprTrait_<const MT2> >;
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.This macro encapsulates the default way o...
Definition: Exception.h:235
Header file for auxiliary alias declarations.
Compile time check whether the given type is a computational expression template.This type trait clas...
Definition: IsComputation.h:72
Header file for mathematical functions.
constexpr bool useOptimizedKernels
Configuration switch for optimized kernels.This configuration switch enables/disables all optimized c...
Definition: Optimizations.h:84
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition: DMatTSMatMultExpr.h:247
Header file for the Rows type trait.
Header file for the IsUniUpper type trait.
const DMatDMatMultExpr< T1, T2 > operator*(const DenseMatrix< T1, false > &lhs, const DenseMatrix< T2, false > &rhs)
Multiplication operator for the multiplication of two row-major dense matrices ( ).
Definition: DMatDMatMultExpr.h:7800
Compile time check for triangular matrix types.This type trait tests whether or not the given templat...
Definition: IsTriangular.h:87
Header file for basic type definitions.
ResultType_< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:133
EnableIf_< IsDenseMatrix< MT1 > > smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition: DenseMatrix.h:160
Header file for the IsSparseMatrix type trait.
Header file for the serial shim.
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition: DMatTSMatMultExpr.h:385
Header file for the IsDiagonal type trait.
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a dense, N-dimensional matrix type...
Definition: DenseMatrix.h:61
Header file for the ColumnExprTrait class template.
BLAZE_ALWAYS_INLINE MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition: Matrix.h:188
Header file for the IsColumnMajorMatrix type trait.
void reset(const DiagonalProxy< MT > &proxy)
Resetting the represented element to the default initial values.
Definition: DiagonalProxy.h:533
Header file for the IsRowVector type trait.
typename DisableIf< Condition, T >::Type DisableIf_
Auxiliary type for the DisableIf class template.The DisableIf_ alias declaration provides a convenien...
Definition: DisableIf.h:223
Header file for the And class template.
LeftOperand leftOperand() const noexcept
Returns the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:331
const ElementType_< MT > min(const DenseMatrix< MT, SO > &dm)
Returns the smallest element of the dense matrix.
Definition: DenseMatrix.h:1669
Compile time check for lower triangular matrices.This type trait tests whether or not the given templ...
Definition: IsLower.h:88
const DMatSerialExpr< MT, SO > serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition: DMatSerialExpr.h:723
typename MultTrait< T1, T2 >::Type MultTrait_
Auxiliary alias declaration for the MultTrait class template.The MultTrait_ alias declaration provide...
Definition: MultTrait.h:245
Header file for the Computation base class.
Header file for the MatMatMultExpr base class.
Compile time check for upper triangular matrices.This type trait tests whether or not the given templ...
Definition: IsUpper.h:88
Constraints on the storage order of matrix types.
Header file for the RequiresEvaluation type trait.
System settings for performance optimizations.
Header file for the IsUniLower type trait.
typename T::ResultType ResultType_
Alias declaration for nested ResultType type definitions.The ResultType_ alias declaration provides a...
Definition: Aliases.h:323
const ElementType_< MT > max(const DenseMatrix< MT, SO > &dm)
Returns the largest element of the dense matrix.
Definition: DenseMatrix.h:1716
EnableIf_< IsDenseMatrix< MT1 > > smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:129
ResultType_< MT2 > RT2
Result type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:134
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, ColumnExprTrait_< MT > > column(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific column of the given matrix.
Definition: Column.h:126
Base class for dense matrices.The DenseMatrix class is a base class for all dense matrix classes...
Definition: DenseMatrix.h:70
Base class for sparse matrices.The SparseMatrix class is a base class for all sparse matrix classes...
Definition: Forward.h:109
Constraint on the data type.
const ResultType CompositeType
Data type for composite expression templates.
Definition: DMatTSMatMultExpr.h:202
typename IfTrue< Condition, T1, T2 >::Type IfTrue_
Auxiliary alias declaration for the IfTrue class template.The IfTrue_ alias declaration provides a co...
Definition: If.h:109
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition: DMatTSMatMultExpr.h:375
Constraint on the data type.
Constraint on the data type.
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition: DMatTSMatMultExpr.h:321
Header file for the MultExprTrait class template.
Compile time check to query the requirement to evaluate an expression.Via this type trait it is possi...
Definition: RequiresEvaluation.h:72
SubvectorExprTrait_< VT, unaligned > subvector(Vector< VT, TF > &vector, size_t index, size_t size)
Creating a view on a specific subvector of the given vector.
Definition: Subvector.h:152
typename T::CompositeType CompositeType_
Alias declaration for nested CompositeType type definitions.The CompositeType_ alias declaration prov...
Definition: Aliases.h:83
Header file for the DisableIf class template.
Header file for the multiplication trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Namespace of the Blaze C++ math library.
Definition: Blaze.h:57
Header file for the If class template.
Compile time check for row-major matrix types.This type trait tests whether or not the given template...
Definition: IsRowMajorMatrix.h:83
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a column-major dense or sparse matri...
Definition: ColumnMajorMatrix.h:61
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition: DMatTSMatMultExpr.h:365
MultTrait_< RT1, RT2 > ResultType
Result type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:197
const Element * ConstIterator
Iterator over constant elements.
Definition: CompressedMatrix.h:2647
CompositeType_< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:137
EnableIf_< IsDenseMatrix< MT1 > > smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs)
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition: DenseMatrix.h:98
Header file for the Or class template.
Header file for the TDVecTSMatMultExprTrait class template.
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.This macro encapsulates the default way of Bl...
Definition: Exception.h:331
Header file for the DenseMatrix base class.
IfTrue_< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side sparse matrix operand.
Definition: DMatTSMatMultExpr.h:214
Header file for the Columns type trait.
Header file for the TSMatDVecMultExprTrait class template.
OppositeType_< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition: DMatTSMatMultExpr.h:198
typename T::ElementType ElementType_
Alias declaration for nested ElementType type definitions.The ElementType_ alias declaration provides...
Definition: Aliases.h:163
Header file for the DMatDVecMultExprTrait class template.
Header file for the IsLower type trait.
Header file for the IsAligned type trait.
Compile time check for diagonal matrices.This type trait tests whether or not the given template para...
Definition: IsDiagonal.h:90
TransposeType_< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:199
#define BLAZE_CONSTRAINT_MUST_BE_REFERENCE_TYPE(T)
Constraint on the data type.In case the given data type T is not a reference type, a compilation error is created.
Definition: Reference.h:60
Header file for the IsTriangular type trait.
IfTrue_< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition: DMatTSMatMultExpr.h:211
Constraints on the storage order of matrix types.
Header file for the exception macros of the math module.
Compile time check for strictly upper triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyUpper.h:86
BLAZE_ALWAYS_INLINE MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition: Matrix.h:254
ElementType_< RT2 > ET2
Element type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:136
RightOperand rhs_
Right-hand side sparse matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:393
Header file for the RowExprTrait class template.
Header file for all forward declarations for expression class templates.
Header file for the IsDenseMatrix type trait.
Header file for the EnableIf class template.
Header file for the IsStrictlyLower type trait.
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.In case the given data types T1 and T2 do not form a valid matrix/matrix ...
Definition: MatMatMultExpr.h:109
If_< IsExpression< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:205
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition: DMatTSMatMultExpr.h:311
ElementType_< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition: DMatTSMatMultExpr.h:135
DisableIf_< Or< IsComputation< MT >, IsTransExpr< MT > >, RowExprTrait_< MT > > row(Matrix< MT, SO > &matrix, size_t index)
Creating a view on a specific row of the given matrix.
Definition: Row.h:126
Base class for all matrix/matrix multiplication expression templates.The MatMatMultExpr class serves ...
Definition: MatMatMultExpr.h:65
Header file for the IsSparseVector type trait.
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is a symmetric matrix type, a compilation error is created.
Definition: Symmetric.h:79
Header file for the SubmatrixExprTrait class template.
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a row-major dense or sparse matrix t...
Definition: RowMajorMatrix.h:61
Header file for run time assertion macros.
Utility type for generic codes.
const ElementType ReturnType
Return type for expression template evaluations.
Definition: DMatTSMatMultExpr.h:201
typename If< T1, T2, T3 >::Type If_
Auxiliary alias declaration for the If class template.The If_ alias declaration provides a convenient...
Definition: If.h:160
ElementType_< ResultType > ElementType
Resulting element type.
Definition: DMatTSMatMultExpr.h:200
Header file for the reset shim.
If_< IsExpression< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:208
Expression object for dense matrix-transpose sparse matrix multiplications.The DMatTSMatMultExpr clas...
Definition: DMatTSMatMultExpr.h:127
Constraints on the storage order of matrix types.
IntegralConstant< bool, B > BoolConstant
Generic wrapper for a compile time constant boolean value.The BoolConstant class template represents ...
Definition: IntegralConstant.h:100
Header file for the RemoveReference type trait.
typename T::OppositeType OppositeType_
Alias declaration for nested OppositeType type definitions.The OppositeType_ alias declaration provid...
Definition: Aliases.h:243
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.In case either of the two given data types T1 or T2 is not a matrix type ...
Definition: StorageOrder.h:84
Header file for the IsDenseVector type trait.
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition: DMatTSMatMultExpr.h:295
Compile time check for strictly lower triangular matrices.This type trait tests whether or not the gi...
Definition: IsStrictlyLower.h:86
typename T::ConstIterator ConstIterator_
Alias declaration for nested ConstIterator type definitions.The ConstIterator_ alias declaration prov...
Definition: Aliases.h:103
DMatTSMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the DMatTSMatMultExpr class.
Definition: DMatTSMatMultExpr.h:232
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition: DMatTSMatMultExpr.h:392
Header file for the IsRowMajorMatrix type trait.
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition: DMatTSMatMultExpr.h:353
const DMatTransExpr< MT,!SO > trans(const DenseMatrix< MT, SO > &dm)
Calculation of the transpose of the given dense matrix.
Definition: DMatTransExpr.h:950
CompositeType_< MT2 > CT2
Composite type of the right-hand side sparse matrix expression.
Definition: DMatTSMatMultExpr.h:138
Header file for the IsComputation type trait class.
Base class for all compute expression templates.The Computation class serves as a tag for all computa...
Definition: Computation.h:59
Header file for the TDVecDMatMultExprTrait class template.
#define BLAZE_FUNCTION_TRACE
Function trace macro.This macro can be used to reliably trace function calls. In case function tracin...
Definition: FunctionTrace.h:157
Header file for the IntegralConstant class template.
Header file for the TSVecDMatMultExprTrait class template.
typename T::TransposeType TransposeType_
Alias declaration for nested TransposeType type definitions.The TransposeType_ alias declaration prov...
Definition: Aliases.h:403
Header file for the IsUpper type trait.
Header file for the DMatSVecMultExprTrait class template.
Header file for the IsColumnVector type trait.
Constraint on the data type.
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose sparse matrix operand.
Definition: DMatTSMatMultExpr.h:341
Header file for the IsResizable type trait.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.In case of an invalid run time expression, the program execution is terminated. The BLAZE_INTERNAL_ASSERT macro can be disabled by setting the BLAZE_USER_ASSERTION flag to zero or by defining NDEBUG during the compilation.
Definition: Assert.h:101
DMatTSMatMultExpr< MT1, MT2 > This
Type of this DMatTSMatMultExpr instance.
Definition: DMatTSMatMultExpr.h:196
#define BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE(T)
Constraint on the data type.In case the given data type T is not a sparse, N-dimensional matrix type...
Definition: SparseMatrix.h:61
Header file for the IsExpression type trait class.
Header file for the TSMatSVecMultExprTrait class template.
Header file for the FunctionTrace class.