Lock access to S.cache

Instead of using cgo we call the syscall for BPF directly from go.  The
API hasn't changed, however, and we also closely follow the
C-implementation as given in bpf(2).

Not sure if this pure go variant is beneficial.  Manual maintenance of
all constants and structs upon changes of the BPF API would be necessary
and cumbersome.

We would at least need to complement this with auto-generation of
constants and fields from /usr/include/linux/bpf.h.

GetRuntimeAddresses/ebpf/ebpf.go

@@ -1,149 +1,304 @@
+// +build !withcgo
+
 package ebpf
 
-/*
-#include <stdlib.h>
-#include <stdint.h>
-#include <unistd.h>
-#include <linux/unistd.h>
-#include <linux/bpf.h>
-
-// See the bpf man page for details on the following functions
-
-static int bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size)
-{
-    return syscall(__NR_bpf, cmd, attr, size);
-}
-
-static int bpf_create_map(
-	 enum bpf_map_type map_type, unsigned int key_size, unsigned int value_size,
-	 unsigned int max_entries)
-{
-	union bpf_attr attr = {
-		.map_type    = map_type,
-		.key_size    = key_size,
-		.value_size  = value_size,
-		.max_entries = max_entries};
-
-	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
-}
-
-int bpf_get_next_key(int fd, const void *key, void *next_key)
-{
-	union bpf_attr attr = {
-		.map_fd   = fd,
-		.key      = (__u64) (unsigned long) key,
-		.next_key = (__u64) (unsigned long) next_key};
-
-	return bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
-}
-
-
-int bpf_lookup_elem(int fd, const void *key, void *value)
-{
-	union bpf_attr attr = {
-		.map_fd = fd,
-		.key    = (__u64) (unsigned long) key,
-		.value  = (__u64) (unsigned long) value,
-	};
-
-	return bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
-}
-
-int bpf_update_elem(int fd, const void *key, const void *value, uint64_t flags)
-{
-	union bpf_attr attr = {
-		.map_fd = fd,
-		.key    = (__u64) (unsigned long) key,
-		.value  = (__u64) (unsigned long) value,
-		.flags  = flags,
-	};
-
-	return bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
-}
-
-*/
-import "C"
 import (
 	"fmt"
 	"syscall"
 	"unsafe"
 )
 
+// All constants are taken from /usr/include/linux/bpf.h
+
+const BPF_SYSCALL = 321
+
+type bpf_cmd int
+
+const (
+	BPF_MAP_CREATE bpf_cmd = iota
+	BPF_MAP_LOOKUP_ELEM
+	BPF_MAP_UPDATE_ELEM
+	BPF_MAP_DELETE_ELEM
+	BPF_MAP_GET_NEXT_KEY
+	BPF_PROG_LOAD
+	BPF_OBJ_PIN
+	BPF_OBJ_GET
+	BPF_PROG_ATTACH
+	BPF_PROG_DETACH
+	BPF_PROG_TEST_RUN
+	BPF_PROG_GET_NEXT_ID
+	BPF_MAP_GET_NEXT_ID
+	BPF_PROG_GET_FD_BY_ID
+	BPF_MAP_GET_FD_BY_ID
+	BPF_OBJ_GET_INFO_BY_FD
+	BPF_PROG_QUERY
+	BPF_RAW_TRACEPOINT_OPEN
+	BPF_BTF_LOAD
+	BPF_BTF_GET_FD_BY_ID
+	BPF_TASK_FD_QUERY
+	BPF_MAP_LOOKUP_AND_DELETE_ELEM
+	BPF_MAP_FREEZE
+)
+
+type bpf_map_type int
+
+const (
+	BPF_MAP_TYPE_UNSPEC bpf_map_type = iota
+	BPF_MAP_TYPE_HASH
+	BPF_MAP_TYPE_ARRAY
+	BPF_MAP_TYPE_PROG_ARRAY
+	BPF_MAP_TYPE_PERF_EVENT_ARRAY
+	BPF_MAP_TYPE_PERCPU_HASH
+	BPF_MAP_TYPE_PERCPU_ARRAY
+	BPF_MAP_TYPE_STACK_TRACE
+	BPF_MAP_TYPE_CGROUP_ARRAY
+	BPF_MAP_TYPE_LRU_HASH
+	BPF_MAP_TYPE_LRU_PERCPU_HASH
+	BPF_MAP_TYPE_LPM_TRIE
+	BPF_MAP_TYPE_ARRAY_OF_MAPS
+	BPF_MAP_TYPE_HASH_OF_MAPS
+	BPF_MAP_TYPE_DEVMAP
+	BPF_MAP_TYPE_SOCKMAP
+	BPF_MAP_TYPE_CPUMAP
+	BPF_MAP_TYPE_XSKMAP
+	BPF_MAP_TYPE_SOCKHASH
+	BPF_MAP_TYPE_CGROUP_STORAGE
+	BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
+	BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
+	BPF_MAP_TYPE_QUEUE
+	BPF_MAP_TYPE_STACK
+	BPF_MAP_TYPE_SK_STORAGE
+)
+
 // MapFD is a file descriptor representing a eBPF map
-type MapFD int
+type MapFD uint32
+
+/*
+
+All methods in this file implement a syscall to bpf one way or another.  We
+follow the C-API given in bpf(2) and have the corresponding C-functions
+embedded as comments.  Those all refer to the C-function bpf(), which is a
+wrapper for the syscall:
+
+  static int bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size)
+  {
+      return syscall(__NR_bpf, cmd, attr, size);
+  }
+
+Each ouf our methods calls the syscall directly, instead.
+
+*/
 
 // CreateMap creates an eBPF map from int->uint64. The file descriptor of the
 // created map is returned.
 func CreateMap() (MapFD, error) {
+	/*
+	   static int bpf_create_map(
+	   	 enum bpf_map_type map_type, unsigned int key_size, unsigned int value_size,
+	   	 unsigned int max_entries)
+	   {
+	   	union bpf_attr attr = {
+	   		.map_type    = map_type,
+	   		.key_size    = key_size,
+	   		.value_size  = value_size,
+	   		.max_entries = max_entries};
 
-	r, errno := C.bpf_create_map(C.BPF_MAP_TYPE_HASH, 4, 8, 64)
+	   	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
+	   }
+	*/
 
-	if r == -1 {
-		return MapFD(r), fmt.Errorf("bpf_create_map with errno %d", errno)
+	create_attr := struct {
+		map_type    uint32
+		key_size    uint32
+		value_size  uint32
+		max_entries uint32
+	}{
+		map_type:    uint32(BPF_MAP_TYPE_HASH),
+		key_size:    4,
+		value_size:  8,
+		max_entries: 64,
 	}
 
+	r, _, err := syscall.Syscall(
+		BPF_SYSCALL,
+		uintptr(BPF_MAP_CREATE),
+		uintptr(unsafe.Pointer(&create_attr)),
+		unsafe.Sizeof(create_attr),
+	)
+
+	if err != 0 {
+		return 0, err
+	}
 	return MapFD(r), nil
 }
 
+func (mfd MapFD) bpf_get_next_key(key *int, next_key *int) error {
+	/*
+	   int bpf_get_next_key(int fd, const void *key, void *next_key)
+	   {
+	   	union bpf_attr attr = {
+	   		.map_fd   = fd,
+	   		.key      = (__u64) (unsigned long) key,
+	   		.next_key = (__u64) (unsigned long) next_key};
+
+	   	return bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
+	   }
+	*/
+
+	next_attr := struct {
+		map_fd   MapFD
+		key      uint64
+		next_key uint64
+	}{
+		map_fd:   mfd,
+		key:      uint64(uintptr(unsafe.Pointer(key))),
+		next_key: uint64(uintptr(unsafe.Pointer(next_key))),
+	}
+
+	r, _, err := syscall.Syscall(
+		BPF_SYSCALL,
+		uintptr(BPF_MAP_GET_NEXT_KEY),
+		uintptr(unsafe.Pointer(&next_attr)),
+		unsafe.Sizeof(next_attr),
+	)
+
+	if r != 0 {
+		return err
+	} else {
+		return nil
+	}
+}
+
+func (mfd MapFD) bpf_lookup_elem(key *int, value *uint64) error {
+	/*
+	   int bpf_lookup_elem(int fd, const void *key, void *value)
+	   {
+	   	union bpf_attr attr = {
+	   		.map_fd = fd,
+	   		.key    = (__u64) (unsigned long) key,
+	   		.value  = (__u64) (unsigned long) value,
+	   	};
+
+	   	return bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
+	   }
+	*/
+
+	lookup_attr := struct {
+		map_fd MapFD
+		key    uint64
+		value  uint64
+	}{
+		map_fd: mfd,
+		key:    uint64(uintptr(unsafe.Pointer(key))),
+		value:  uint64(uintptr(unsafe.Pointer(value))),
+	}
+
+	r, _, err := syscall.Syscall(
+		BPF_SYSCALL,
+		uintptr(BPF_MAP_LOOKUP_ELEM),
+		uintptr(unsafe.Pointer(&lookup_attr)),
+		unsafe.Sizeof(lookup_attr),
+	)
+
+	if r != 0 {
+		return err
+	}
+	return nil
+}
+
 // GetMap gets the key/value pairs from the specified eBPF map as a Go map
 func (mfd MapFD) GetMap() (map[int]uint64, error) {
 	retMap := make(map[int]uint64)
 
-	mapFDC := C.int(mfd)
-	keyC := C.int(0)
-	nextKeyC := C.int(0)
-	valC := C.uint64_t(0)
+	var (
+		key, next int
+		value     uint64
+	)
 
 	for {
-		r, errno := C.bpf_get_next_key(mapFDC, unsafe.Pointer(&keyC), unsafe.Pointer(&nextKeyC))
-		if r == -1 {
-			if errno == syscall.ENOENT {
+		err := mfd.bpf_get_next_key(&key, &next)
+		if err != nil {
+			if err == syscall.ENOENT {
 				// The provided key was the last element. We're done iterating.
 				return retMap, nil
 			}
-			return nil, fmt.Errorf("bpf_get_next_key failed with errno %d", errno)
+			return nil, fmt.Errorf("bpf_get_next_key failed with error %v", err)
 		}
 
-		r = C.bpf_lookup_elem(mapFDC, unsafe.Pointer(&nextKeyC), unsafe.Pointer(&valC))
-		if r == -1 {
-			return nil, fmt.Errorf("bpf_lookup_elem failed")
+		err = mfd.bpf_lookup_elem(&next, &value)
+		if err != nil {
+			return nil, fmt.Errorf("bpf_lookup_elem failed with error %v", err)
 		}
-		retMap[int(nextKeyC)] = uint64(valC)
 
-		keyC = nextKeyC
+		retMap[next] = value
+
+		key = next
 	}
 }
 
+const (
+	BPF_ANY = iota
+	BPF_NOEXIST
+	BPF_EXIST
+	BPF_F_LOCK
+)
+
 // Add puts the (key, value) into the eBPF map, only if the key does not exist
 // yet in the map. It returns an error otherwise.
 func (mfd MapFD) Add(key int, value uint64) error {
-	return mfd.updateElement(key, value, C.BPF_NOEXIST)
+	return mfd.updateElement(key, value, BPF_NOEXIST)
 }
 
 // Change changes the value to an existing key in the eBPF map.  It returns an
 // error otherwise.
 func (mfd MapFD) Change(key int, value uint64) error {
-	return mfd.updateElement(key, value, C.BPF_EXIST)
+	return mfd.updateElement(key, value, BPF_EXIST)
 }
 
 // Set puts the (key, value) into the eBPF map.  It will create or overwrite an
 // existing entry for that key.
 func (mfd MapFD) Set(key int, value uint64) error {
-	return mfd.updateElement(key, value, C.BPF_ANY)
+	return mfd.updateElement(key, value, BPF_ANY)
 }
 
 // updateElement is the low level wrapper to bpf_update_elem, used from Add(),
 // Set() and Change().
-func (mfd MapFD) updateElement(key int, value uint64, flag C.uint64_t) error {
+func (mfd MapFD) updateElement(key int, value uint64, flag uint64) error {
+	/*
+	   int bpf_update_elem(int fd, const void *key, const void *value, uint64_t flags)
+	   {
+	   	union bpf_attr attr = {
+	   		.map_fd = fd,
+	   		.key    = (__u64) (unsigned long) key,
+	   		.value  = (__u64) (unsigned long) value,
+	   		.flags  = flags,
+	   	};
 
-	r, errno := C.bpf_update_elem(C.int(mfd),
-		unsafe.Pointer(&key),
-		unsafe.Pointer(&value),
-		flag)
+	   	return bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
+	   }
+	*/
 
-	if r == -1 {
-		return fmt.Errorf("bpf_update_elem failed with errno %d", errno)
+	update_attr := struct {
+		map_fd MapFD
+		key    uint64
+		value  uint64
+		flags  uint64
+	}{
+		map_fd: mfd,
+		key:    uint64(uintptr(unsafe.Pointer(&key))),
+		value:  uint64(uintptr(unsafe.Pointer(&value))),
+		flags:  flag,
 	}
 
+	r, _, err := syscall.Syscall(
+		BPF_SYSCALL,
+		uintptr(BPF_MAP_UPDATE_ELEM),
+		uintptr(unsafe.Pointer(&update_attr)),
+		unsafe.Sizeof(update_attr),
+	)
+
+	if r != 0 || err != 0 {
+		return fmt.Errorf("couldn't update element: %s", err.Error())
+	}
 	return nil
 }

GetRuntimeAddresses/ebpf/ebpf_cgo.go

@@ -0,0 +1,151 @@
+// +build withcgo
+
+package ebpf
+
+/*
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <linux/unistd.h>
+#include <linux/bpf.h>
+
+// See the bpf man page for details on the following functions
+
+static int bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size)
+{
+    return syscall(__NR_bpf, cmd, attr, size);
+}
+
+static int bpf_create_map(
+	 enum bpf_map_type map_type, unsigned int key_size, unsigned int value_size,
+	 unsigned int max_entries)
+{
+	union bpf_attr attr = {
+		.map_type    = map_type,
+		.key_size    = key_size,
+		.value_size  = value_size,
+		.max_entries = max_entries};
+
+	return bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
+}
+
+int bpf_get_next_key(int fd, const void *key, void *next_key)
+{
+	union bpf_attr attr = {
+		.map_fd   = fd,
+		.key      = (__u64) (unsigned long) key,
+		.next_key = (__u64) (unsigned long) next_key};
+
+	return bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
+}
+
+
+int bpf_lookup_elem(int fd, const void *key, void *value)
+{
+	union bpf_attr attr = {
+		.map_fd = fd,
+		.key    = (__u64) (unsigned long) key,
+		.value  = (__u64) (unsigned long) value,
+	};
+
+	return bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
+}
+
+int bpf_update_elem(int fd, const void *key, const void *value, uint64_t flags)
+{
+	union bpf_attr attr = {
+		.map_fd = fd,
+		.key    = (__u64) (unsigned long) key,
+		.value  = (__u64) (unsigned long) value,
+		.flags  = flags,
+	};
+
+	return bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
+}
+
+*/
+import "C"
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+)
+
+// MapFD is a file descriptor representing a eBPF map
+type MapFD int
+
+// CreateMap creates an eBPF map from int->uint64. The file descriptor of the
+// created map is returned.
+func CreateMap() (MapFD, error) {
+
+	r, errno := C.bpf_create_map(C.BPF_MAP_TYPE_HASH, 4, 8, 64)
+
+	if r == -1 {
+		return MapFD(r), fmt.Errorf("bpf_create_map with errno %d", errno)
+	}
+
+	return MapFD(r), nil
+}
+
+// GetMap gets the key/value pairs from the specified eBPF map as a Go map
+func (mfd MapFD) GetMap() (map[int]uint64, error) {
+	retMap := make(map[int]uint64)
+
+	mapFDC := C.int(mfd)
+	keyC := C.int(0)
+	nextKeyC := C.int(0)
+	valC := C.uint64_t(0)
+
+	for {
+		r, errno := C.bpf_get_next_key(mapFDC, unsafe.Pointer(&keyC), unsafe.Pointer(&nextKeyC))
+		if r == -1 {
+			if errno == syscall.ENOENT {
+				// The provided key was the last element. We're done iterating.
+				return retMap, nil
+			}
+			return nil, fmt.Errorf("bpf_get_next_key failed with errno %d", errno)
+		}
+
+		r = C.bpf_lookup_elem(mapFDC, unsafe.Pointer(&nextKeyC), unsafe.Pointer(&valC))
+		if r == -1 {
+			return nil, fmt.Errorf("bpf_lookup_elem failed")
+		}
+		retMap[int(nextKeyC)] = uint64(valC)
+
+		keyC = nextKeyC
+	}
+}
+
+// Add puts the (key, value) into the eBPF map, only if the key does not exist
+// yet in the map. It returns an error otherwise.
+func (mfd MapFD) Add(key int, value uint64) error {
+	return mfd.updateElement(key, value, C.BPF_NOEXIST)
+}
+
+// Change changes the value to an existing key in the eBPF map.  It returns an
+// error otherwise.
+func (mfd MapFD) Change(key int, value uint64) error {
+	return mfd.updateElement(key, value, C.BPF_EXIST)
+}
+
+// Set puts the (key, value) into the eBPF map.  It will create or overwrite an
+// existing entry for that key.
+func (mfd MapFD) Set(key int, value uint64) error {
+	return mfd.updateElement(key, value, C.BPF_ANY)
+}
+
+// updateElement is the low level wrapper to bpf_update_elem, used from Add(),
+// Set() and Change().
+func (mfd MapFD) updateElement(key int, value uint64, flag C.uint64_t) error {
+
+	r, errno := C.bpf_update_elem(C.int(mfd),
+		unsafe.Pointer(&key),
+		unsafe.Pointer(&value),
+		flag)
+
+	if r == -1 {
+		return fmt.Errorf("bpf_update_elem failed with errno %d", errno)
+	}
+
+	return nil
+}

GetRuntimeAddresses/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/optimyze-interviews/OezguerKesim/GetRuntimeAddresses/ebpf"
 	"github.com/optimyze-interviews/OezguerKesim/GetRuntimeAddresses/symbolyze"
@@ -28,11 +29,14 @@ func main() {
 
 	scanner := symbolyze.NewScanner(*symbol, *glob)
 	scanner.OnFound(mapFD.Set)
-	if *debug {
-		scanner.DebugOn()
-	}
+	scanner.Debug(*debug)
 
-	err = scanner.Run()
+	go scanner.RunEvery(time.Second)
+
+	time.Sleep(10 * time.Second)
+	scanner.Stop()
+
+	err = scanner.Errors()
 	if err != nil {
 		fmt.Printf("Failed to run the symbolyze scanner: %s", err)
 		os.Exit(1)

GetRuntimeAddresses/symbolyze/doc.go

@@ -0,0 +1,26 @@
+// Copyright note...
+/*
+Package symbolyze provides a mechanism to search for all occurences of a
+certain symbol in certain mmap'ed ELF binaries of all running processes.
+
+NewScanner(symbol string, glob string) will setup a Scanner, which will search
+for the given symbol name in all mmap'ed ELF-files that match the given
+glob-pattern. The glob-pattern is a shell file name pattern, see
+filepath.Match.
+
+The scanner should be populated with callback functions via calls to OnFound(),
+before starting the scan by calling Run().
+
+For example:
+
+	scanner := symbolyze.NewScanner("_PyRuntime", "*python3*")
+
+	scanner.OnFound(func(pid int, offset uint64) error {
+		fmt.Println("found symbol in", pid, "in offset", offset")
+		return nil
+	})
+
+	err := scanner.Run()
+
+*/
+package symbolyze

GetRuntimeAddresses/symbolyze/gdb_test.go

@@ -15,7 +15,7 @@ import (
 // test-binary `simple` that is linked against python3.7.  If this fails, make
 // necessary adjustments to the Makefile and/or your environment and try again.
 func buildSimple() error {
-	cmd := exec.Command("make")
+	cmd := exec.Command("make", "-s")
 	cmd.Dir = "testdata"
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -115,5 +115,6 @@ func extractOffsetWithGdb(pid int, t *testing.T) (offset uint64, err error) {
 		}
 	}
 
-	return 0, fmt.Errorf("Symbol not found with gdb")
+	return 0, fmt.Errorf("Symbol not found with gdb.\n" +
+		"Maybe /proc/sys/kernel/yama/ptrace_scope must be set to 0? Or try to run the test as root!")
 }

GetRuntimeAddresses/symbolyze/symbolyze.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
 // Scanner represents an engine for scanning for a specific symbol in all
@@ -24,34 +25,60 @@ import (
 // /proc.  Whenever a match is found, all observers will be called with the
 // (pid, offset), concurrently.
 type Scanner struct {
+	rwmutex
+
 	symbol    string
 	pathglob  string
 	cache     map[string]uint64 // Contains (pathname, offset)
 	observers []Observer        // Callbacks
 
-	logger // Embedded logger
-
 	// Instead of using a boolean to indicate debugging, we use function
 	// members.  This way we can populate them with noop-functions in the
 	// non-debug case and not polute the code with if-statements.
 	debugf  func(format string, v ...interface{})
 	debugln func(v ...interface{})
+	logger  // Embedded logger
 
-	err error // error state of the scanner.
+	errors errors
+
+	ticker *time.Ticker // Used to run the scanner repeatedly
+}
+
+type errors []error
+
+func (e errors) Error() string {
+	switch len(e) {
+	case 0:
+		return "nil"
+	case 1:
+		return e[0].Error()
+	default:
+		list := make([]string, len(e))
+		for i := range e {
+			list[i] = e[i].Error()
+		}
+		return fmt.Sprintf("multiple errors:\n%s", strings.Join(list, "\n"))
+	}
 }
 
 // We use a lowercase type alias for *log.Logger so that we can embedd it in
 // Scanner without exporting it.
 type logger = *log.Logger
+type rwmutex = sync.RWMutex
 
-// An Observer is a callback that can be registerd with Scanner.OnFound.  It
+var (
+	nodebugf  = func(format string, v ...interface{}) {}
+	nodebugln = func(v ...interface{}) {}
+)
+
+// An Observer is a callback that can be registered with Scanner.OnFound.  It
 // will be called with a pid and an offset.  Observers are called concurrently.
 // They have to be thread-safe.
 type Observer func(pid int, offset uint64) error
 
 // NewScanner returns a new Scanner that scans all running processes for the
 // given symbol name in all memory-mapped files matching the given pathglob.
-// To be useful, one or more Observer functions should be registerd with
+// To be useful, one or more Observer functions should be registered with
 // Scanner.OnFound(). The scanning starts with a call of Scanner.Run().
 func NewScanner(symbol, pathglob string) *Scanner {
 	return &Scanner{
@@ -62,62 +89,97 @@ func NewScanner(symbol, pathglob string) *Scanner {
 		logger: log.New(os.Stderr, "[symbolyze] ", log.Ltime|log.Lmicroseconds),
 
 		// debugging is off per default.
-		debugf:  func(string, ...interface{}) {},
-		debugln: func(...interface{}) {},
+		debugf:  nodebugf,
+		debugln: nodebugln,
 	}
 }
 
-// Debug sets the scanner into debugging mode.  It must called only once before
-// a call to Scanner.Run().
-func (S *Scanner) DebugOn() {
-	// Use the embedded *log.Logger for debugging.
-	S.debugf = S.Printf
-	S.debugln = S.Println
-	S.debugln("starting in debug-mode")
+// Debug sets the scanner into debugging mode.  It must be called only once
+// before a call to Scanner.Run().
+func (S *Scanner) Debug(on bool) {
+	S.Lock()
+	defer S.Unlock()
+
+	if on {
+		// Use the embedded *log.Logger for debugging.
+		S.debugf = S.Printf
+		S.debugln = S.Println
+		S.debugln("starting in debug-mode")
+	} else {
+		S.debugf = nodebugf
+		S.debugln = nodebugln
+	}
 }
 
 // setErrorf puts the Scanner into an error state with the given error
-// statement.  It also logs the error.
+// statement.  It also logs the error.  setErrorf is not thread-safe.
 func (S *Scanner) setErrorf(format string, a ...interface{}) {
-	S.err = fmt.Errorf(format, a...)
+	S.Lock()
+	S.errors = append(S.errors, fmt.Errorf(format, a...))
+	S.Unlock()
 	S.Printf(format, a...)
 }
 
-// OnFound puts an Observer function into the interal queue.  The functions are
-// called in sequence in their own goroutine whenever the scanner finds the
-// symbol in the a running program.  That implies that an Observer has to be
-// thread-safe.  Errors from the observers will be logged.
-//
-// Calling OnFound is not thread-safe.
-func (S *Scanner) OnFound(fun Observer) {
-	S.observers = append(S.observers, fun)
+func (S *Scanner) HasErrors() bool {
+	S.RLock()
+	defer S.RUnlock()
+	return len(S.errors) > 0
+}
+
+func (S *Scanner) Errors() error {
+	S.RLock()
+	defer S.RUnlock()
+	if len(S.errors) == 0 {
+		return nil
+	} else {
+		e2 := make(errors, len(S.errors))
+		copy(e2, S.errors)
+		return e2
+	}
+}
+
+func (S *Scanner) addError(err error) {
+	S.Lock()
+	defer S.Unlock()
+	S.errors = append(S.errors, err)
+}
+
+// OnFound puts Observer functions at the end of the internal queue.  All
+// Observer functions are called in sequence in their own goroutine whenever
+// the scanner finds the symbol in a running program.  That implies that an
+// Observer has to be thread-safe.  Errors from the observers will be logged.
+func (S *Scanner) OnFound(fun ...Observer) {
+	S.Lock()
+	defer S.Unlock()
+
+	S.observers = append(S.observers, fun...)
 	return
 }
 
-// Run starts the scanning process.  It scans the maps file all processes in
-// /proc for pathnames that match the provided pathglob and that are ELF
-// executables or shared libraries.  It searches for the provided symbol in
-// those files and calls the registered Observer functions concurrently with
+// Run starts the scanning process.  It scans the entries of all /proc/NNN/maps
+// files for pathnames that match the provided path-glob and are executables or
+// shared libraries in ELF format.  It searches for the provided symbol in
+// those files and calls the registered Observer functions, concurrently, with
 // the pid and offset of the symbol.
 //
 // Run will return an error if it couldn't read the proc filesystem.  Otherwise
 // it will try to continue to loop over all pids, writing potential errors to
 // the console.  Errors from the observer functions are logged.
 func (S *Scanner) Run() error {
-	if S.err != nil {
-		return S.err
+	if S.HasErrors() {
+		return S.Errors()
 	}
 
 	proc, err := os.Open("/proc")
 	if err != nil {
 		S.setErrorf("Failed to open /proc: %v\n", err)
-		return S.err
+		return S.Errors()
 	}
 
 	infos, err := proc.Readdir(-1)
 	if err != nil {
 		S.setErrorf("Failed to read /proc: %v\n", err)
-		return S.err
+		return S.Errors()
 	}
 
 	proc.Close()
@@ -132,7 +194,7 @@ func (S *Scanner) Run() error {
 			continue
 		} else if pid, err := strconv.Atoi(pid_s); err != nil {
 			continue
-		} else if offset, found := S.searchSymbolIn(pid); !found {
+		} else if offset, found := S.searchSymbolInPid(pid); !found {
 			continue
 		} else {
 			// Call the observers with (pid, offset), in the
@@ -143,8 +205,7 @@ func (S *Scanner) Run() error {
 					go func() {
 						err = observer(pid, offset)
 						if err != nil {
-							S.Printf("S.observer[%d](%d, %d) error: %v", n, pid, offset, err)
-							// TODO: accumulate errors from all Observers.
+							S.addError(fmt.Errorf("S.observer[%d](%d, %d) error: %v", n, pid, offset, err))
 						}
 						wg.Done()
 					}()
@@ -154,10 +215,31 @@ func (S *Scanner) Run() error {
 	}
 
 	wg.Wait() // Wait for all observers to finish
-	return S.err
+	return S.Errors()
 }
 
-// searchSymbolIn loops over the entries in /proc/<pid>/maps and searches for
+// RunEvery() starts a scanning process and repeats at the given time step.
+func (S *Scanner) RunEvery(step time.Duration) {
+	S.ticker = time.NewTicker(step)
+	for {
+		select {
+		case <-S.ticker.C:
+			err := S.Run()
+			if err != nil {
+				S.Println(err)
+			}
+		}
+	}
+}
+
+func (S *Scanner) Stop() {
+	if S.ticker != nil {
+		S.debugln("Stopping ticker")
+		S.ticker.Stop()
+	}
+}
+
+// searchSymbolInPid loops over the entries in /proc/<pid>/maps and searches for
 // the symbol in the mapped files.
 //
 // The current implementation makes the following assumptions:
@@ -167,12 +249,12 @@ func (S *Scanner) Run() error {
 //   4. The symbol is present at most in one mapped file at the same time.
 //
 // It returns the offsets in memory of the running program, if found.
-func (S *Scanner) searchSymbolIn(pid int) (offset uint64, found bool) {
+func (S *Scanner) searchSymbolInPid(pid int) (offset uint64, found bool) {
 
 	path := filepath.Join("/proc", strconv.Itoa(pid), "maps")
 	maps, err := os.Open(path)
 	if err != nil {
-		S.Printf("%v\n", err)
+		S.debugf("%v\n", err)
 		return 0, false
 	}
 
@@ -230,7 +312,7 @@ func (S *Scanner) searchSymbolIn(pid int) (offset uint64, found bool) {
 		// Finally, find the symbol in the binary.  If found,
 		// findSymbol returns the offset of the symbol in memory,
 		// taking alignment into account.
-		memOffset, found := S.findSymbol(pathname)
+		memOffset, found := S.findSymbolInELF(pathname)
 		if !found {
 			continue
 		}
@@ -242,20 +324,23 @@ func (S *Scanner) searchSymbolIn(pid int) (offset uint64, found bool) {
 	return 0, false
 }
 
-// findSymbol searches for the provided symbol in the given pathname to an
+// findSymbolInELF searches for the provided symbol in the given pathname to an
 // ELF-file.  If found, it returns the offset of the symbol in the virtual
 // memory according to the fomula:
 //
 // 	vmOffset = alignedOffset(section) + offsetInSection(symbol)
 //
-// The result will be cached so that subsequent calls to findSymbol with the
-// same pathname can quickly return.
-func (S *Scanner) findSymbol(pathname string) (offset uint64, found bool) {
+// The result will be cached so that subsequent calls to findSymbolInELF with
+// the same pathname can quickly return.
+func (S *Scanner) findSymbolInELF(pathname string) (offset uint64, found bool) {
 
 	// 0. Return the value from the cache, if found.
+	S.RLock()
 	if offset, found = S.cache[pathname]; found {
+		S.RUnlock()
 		return offset, found
 	}
+	S.RUnlock()
 
 	// 1. Open the file with the ELF-parser
 	file, err := elf.Open(pathname)
@@ -308,7 +393,9 @@ func (S *Scanner) findSymbol(pathname string) (offset uint64, found bool) {
 
 	// 6. Store this calculation in our cache so that we don't to touch
 	// this file again.
+	S.Lock()
 	S.cache[pathname] = vmOffset
+	S.Unlock()
 
 	return vmOffset, true
 }