359 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,  in emitAtomicCmpXchg()
409 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,  in emitAtomicCmpXchgFailureSet()
493 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,  in EmitAtomicOp()
502   case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicOp()
503   case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicOp()
506   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicOp()
507   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicOp()
511   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicOp()
512   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicOp()
516   case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicOp()
517   case AtomicExpr::AO__atomic_compare_exchange_n: {  in EmitAtomicOp()
546   case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicOp()
547   case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicOp()
548   case AtomicExpr::AO__atomic_load_n:  in EmitAtomicOp()
549   case AtomicExpr::AO__atomic_load: {  in EmitAtomicOp()
557   case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicOp()
558   case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicOp()
559   case AtomicExpr::AO__atomic_store:  in EmitAtomicOp()
560   case AtomicExpr::AO__atomic_store_n: {  in EmitAtomicOp()
568   case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicOp()
569   case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicOp()
570   case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicOp()
571   case AtomicExpr::AO__atomic_exchange:  in EmitAtomicOp()
575   case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicOp()
578   case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicOp()
579   case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicOp()
580   case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicOp()
584   case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicOp()
587   case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicOp()
588   case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicOp()
589   case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicOp()
593   case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicOp()
594   case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicOp()
599   case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicOp()
600   case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicOp()
605   case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicOp()
608   case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicOp()
609   case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicOp()
610   case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicOp()
614   case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicOp()
617   case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicOp()
618   case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicOp()
619   case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicOp()
623   case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicOp()
626   case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicOp()
627   case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicOp()
628   case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicOp()
632   case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicOp()
635   case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicOp()
650   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)  in EmitAtomicOp()
665 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,  in EmitAtomicOp()
745 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {  in EmitAtomicExpr()
757   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||  in EmitAtomicExpr()
758       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {  in EmitAtomicExpr()
783   case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
784   case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
787   case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
788   case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
789   case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
792   case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
796   case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
800   case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
805   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
806   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
807   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
808   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
809   case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
810   case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
812     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)  in EmitAtomicExpr()
817     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||  in EmitAtomicExpr()
818         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)  in EmitAtomicExpr()
822   case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
823   case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
824   case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
825   case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
842   case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
843   case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
844   case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
845   case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
846   case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
847   case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
848   case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
849   case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
850   case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
851   case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
852   case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
853   case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
854   case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
855   case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
856   case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
857   case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
858   case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
859   case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
860   case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
861   case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
862   case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
863   case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
864   case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
865   case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
866   case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
867   case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
868   case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
869   case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
896     case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
897     case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
900     case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
901     case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
902     case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
903     case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
904     case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
905     case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
906     case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
907     case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
908     case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
909     case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
910     case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
911     case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
912     case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
913     case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
914     case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
915     case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
916     case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
917     case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
918     case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
919     case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
920     case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
921     case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
922     case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
923     case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
924     case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
925     case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
930     case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
931     case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
932     case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
933     case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
939     case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
940     case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
941     case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
942     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
943     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
944     case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
945     case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
946     case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
947     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
948     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
949     case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
950     case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
951     case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
952     case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
994     case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
995     case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
1005     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1006     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1007     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1008     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1009     case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
1010     case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
1026     case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
1027     case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
1028     case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
1029     case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
1036     case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
1037     case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
1038     case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
1039     case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
1048     case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
1049     case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
1050     case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
1051     case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
1056     case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
1059     case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
1060     case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
1061     case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
1068     case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
1071     case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
1072     case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
1073     case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
1080     case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
1083     case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
1084     case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
1085     case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
1092     case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
1095     case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
1096     case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
1097     case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
1104     case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
1107     case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
1108     case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
1109     case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
1114     case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
1115     case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
1122     case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
1123     case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
1132     case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
1135     case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
1188       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)  in EmitAtomicExpr()
1204   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||  in EmitAtomicExpr()
1205                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||  in EmitAtomicExpr()
1206                  E->getOp() == AtomicExpr::AO__atomic_store ||  in EmitAtomicExpr()
1207                  E->getOp() == AtomicExpr::AO__atomic_store_n;  in EmitAtomicExpr()
1208   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||  in EmitAtomicExpr()
1209                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||  in EmitAtomicExpr()
1210                 E->getOp() == AtomicExpr::AO__atomic_load ||  in EmitAtomicExpr()
1211                 E->getOp() == AtomicExpr::AO__atomic_load_n;  in EmitAtomicExpr()