Address possible data race in Notifiable: setSuspended

WARNING: ThreadSanitizer: data race (pid=19078)

  Write of size 1 at 0x7d600000f078 by thread T10 (mutexes: write M21717):
    #0 Notifiable::setSuspended(bool) /home/abhinav/couchbase/ep-engine/src/tapconnection.h:764 (ep.so+0x00000005fe6a)
    couchbase#1 TapProducer::suspendedConnection_UNLOCKED(bool) /home/abhinav/couchbase/ep-engine/src/tapconnection.cc:717 (ep.so+0x00000013b24e)
    couchbase#2 ExecutorThread::run() /home/abhinav/couchbase/ep-engine/src/executorthread.cc:112 (ep.so+0x0000000f89f6)
    couchbase#3 launch_executor_thread(void*) /home/abhinav/couchbase/ep-engine/src/executorthread.cc:33 (ep.so+0x0000000f8595)
    couchbase#4 platform_thread_wrap /home/abhinav/couchbase/platform/src/cb_pthreads.c:23 (libplatform.so.0.1.0+0x000000003d31)

  Previous read of size 1 at 0x7d600000f078 by main thread (mutexes: write M21715):
    #0 Notifiable::isSuspended() /home/abhinav/couchbase/ep-engine/src/tapconnection.h:768 (ep.so+0x0000000dfdf2)
    couchbase#1 EventuallyPersistentEngine::walkTapQueue(void const*, void**, void**, unsigned short*, unsigned char*, unsigned short*, unsigned int*, unsigned short*) /home/abhinav/couchbase/ep-engine/src/ep_engine.cc:2531 (ep.so+0x0000000b7ecc)
    couchbase#2 EvpTapIterator(engine_interface*, void const*, void**, void**, unsigned short*, unsigned char*, unsigned short*, unsigned int*, unsigned short*) /home/abhinav/couchbase/ep-engine/src/ep_engine.cc:1440 (ep.so+0x0000000d5ad7)
    couchbase#3 mock_tap_iterator(engine_interface*, void const*, void**, void**, unsigned short*, unsigned char*, unsigned short*, unsigned int*, unsigned short*) /home/abhinav/couchbase/memcached/programs/engine_testapp/engine_testapp.cc:81 (engine_testapp+0x0000000bbda2)
    couchbase#4 test_tap_ack_stream(engine_interface*, engine_interface_v1*) /home/abhinav/couchbase/ep-engine/tests/ep_testsuite.cc:7353 (ep_testsuite.so+0x0000000504a7)
    couchbase#5 execute_test(test, char const*, char const*) /home/abhinav/couchbase/memcached/programs/engine_testapp/engine_testapp.cc:1090 (engine_testapp+0x0000000b946c)
    #6 __libc_start_main /build/buildd/eglibc-2.19/csu/libc-start.c:287 (libc.so.6+0x000000021ec4)

Change-Id: I596c93c048767911b052861193822ca17270a5dd
Reviewed-on: http://review.couchbase.org/55792
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Chiyoung Seo <chiyoung@couchbase.com>

Author: abhinavdangeti

src/tapconnection.h

@@ -760,8 +760,9 @@ class Notifiable {
         return notifySent;
     }
 
-    virtual void setSuspended(bool value) {
-        suspended = value;
+    bool setSuspended(bool val) {
+        bool inverse = !val;
+        return suspended.compare_exchange_strong(inverse, val);
     }
 
     bool isSuspended() {
@@ -770,7 +771,7 @@ class Notifiable {
 
 private:
     //! Is this tap connection in a suspended state
-    bool suspended;
+    AtomicValue<bool> suspended;
     //! Connection is temporarily paused?
     AtomicValue<bool> paused;
     //! Flag indicating if the notification event is scheduled