Merge pull request #179 from geeksville/dev

omg so good

bin/version.sh

@@ -1,3 +1,3 @@
 
 
-export VERSION=0.7.4
+export VERSION=0.7.5

docs/software/TODO.md

@@ -1,9 +1,17 @@
 # High priority
 
+- why is the net so chatty now?
+- do a release
+- device wakes, turns BLE on and phone doesn't notice (while phone was sitting in auto-connect)
+- E22 bringup
+- encryption review findings writeup
+- turn on modem-sleep mode - https://github.com/espressif/arduino-esp32/issues/1142#issuecomment-512428852
+
 # Medium priority
 
 Items to complete before the first beta release.
 
+- turn on watchdog timer (because lib code seems buggy)
 - show battery level as % full
 - rx signal measurements -3 marginal, -9 bad, 10 great, -10 means almost unusable. So scale this into % signal strength. preferably as a graph, with an X indicating loss of comms.
 
@@ -24,6 +32,7 @@ During the beta timeframe the following improvements 'would be nice'
 
 Items after the first final candidate release.
 
+- Change back to using a fixed sized MemoryPool rather than MemoryDynamic (see bug #149)
 - scan to find channels with low background noise? (Use CAD mode of the RF95 to automatically find low noise channels)
 - If the phone doesn't read fromradio mailbox within X seconds, assume the phone is gone and we can stop queing location msgs
   for it (because it will redownload the nodedb when it comes back)
@@ -34,7 +43,7 @@ Items after the first final candidate release.
 - Don't store position packets in the to phone fifo if we are disconnected. The phone will get that info for 'free' when it
   fetches the fresh nodedb.
 - Use the RFM95 sequencer to stay in idle mode most of the time, then automatically go to receive mode and automatically go from transmit to receive mode. See 4.2.8.2 of manual.
-- Use fixed32 for node IDs, packetIDs and lat/lon - will require all nodes to be updated, but make messages slightly smaller.
+- Use fixed32 for node IDs, packetIDs, successid, failid, and lat/lon - will require all nodes to be updated, but make messages slightly smaller.
 - add "store and forward" support for messages, or move to the DB sync model. This would allow messages to be eventually delivered even if nodes are out of contact at the moment.
 - use variable length Strings in protobufs (instead of current fixed buffers). This would save lots of RAM
 - use BLEDevice::setPower to lower our BLE transmit power - extra range doesn't help us, it costs amps and it increases snoopability

platformio.ini

@@ -22,8 +22,6 @@ default_envs = tbeam ; Note: the github actions CI test build can't yet build NR
 ; HW_VERSION (default emptystring)
 
 [env]
-platform = espressif32
-framework = arduino
 
 ; customize the partition table
 ; http://docs.platformio.org/en/latest/platforms/espressif32.html#partition-tables
@@ -79,6 +77,8 @@ lib_deps =
   
 ; Common settings for ESP targes, mixin with extends = esp32_base
 [esp32_base]
+platform = espressif32
+framework = arduino
 src_filter = 
   ${env.src_filter} -<nrf52/>
 upload_speed = 921600
@@ -86,6 +86,8 @@ debug_init_break = tbreak setup
 build_flags =
   ${env.build_flags} -Wall -Wextra -Isrc/esp32 
 lib_ignore = segger_rtt
+platform_packages =
+    framework-arduinoespressif32 @ https://github.com/meshtastic/arduino-esp32.git
 
 ; The 1.0 release of the TBEAM board
 [env:tbeam]

src/WorkerThread.cpp

@@ -1,4 +1,5 @@
 #include "WorkerThread.h"
+#include "debug.h"
 #include <assert.h>
 
 void Thread::start(const char *name, size_t stackSize, uint32_t priority)
@@ -16,6 +17,15 @@ void WorkerThread::doRun()
 {
     while (!wantExit) {
         block();
+
+#ifdef DEBUG_STACK
+        static uint32_t lastPrint = 0;
+        if (millis() - lastPrint > 10 * 1000L) {
+            lastPrint = millis();
+            meshtastic::printThreadInfo("net");
+        }
+#endif
+
         loop();
     }
 }
@@ -28,8 +38,6 @@ void NotifiedWorkerThread::notify(uint32_t v, eNotifyAction action)
     xTaskNotify(taskHandle, v, action);
 }
 
-
-
 void NotifiedWorkerThread::block()
 {
     xTaskNotifyWait(0,                                          // don't clear notification on entry

src/WorkerThread.h

@@ -15,6 +15,8 @@ class Thread
 
     virtual ~Thread() { vTaskDelete(taskHandle); }
 
+    uint32_t getStackHighwaterMark() { return uxTaskGetStackHighWaterMark(taskHandle); }
+
   protected:
     /**
      * The method that will be called when start is called.

src/esp32/BluetoothUtil.cpp

@@ -8,53 +8,6 @@
 
 SimpleAllocator btPool;
 
-/**
- * Create standard device info service
- **/
-BLEService *createDeviceInfomationService(BLEServer *server, std::string hwVendor, std::string swVersion,
-                                          std::string hwVersion = "")
-{
-    BLEService *deviceInfoService = server->createService(BLEUUID((uint16_t)ESP_GATT_UUID_DEVICE_INFO_SVC));
-
-    BLECharacteristic *swC =
-        new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_SW_VERSION_STR), BLECharacteristic::PROPERTY_READ);
-    BLECharacteristic *mfC = new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_MANU_NAME), BLECharacteristic::PROPERTY_READ);
-    // BLECharacteristic SerialNumberCharacteristic(BLEUUID((uint16_t) ESP_GATT_UUID_SERIAL_NUMBER_STR),
-    // BLECharacteristic::PROPERTY_READ);
-
-    /*
-           * Mandatory characteristic for device info service?
-
-          BLECharacteristic *m_pnpCharacteristic = m_deviceInfoService->createCharacteristic(ESP_GATT_UUID_PNP_ID,
-      BLECharacteristic::PROPERTY_READ);
-
-      uint8_t sig, uint16_t vid, uint16_t pid, uint16_t version;
-          uint8_t pnp[] = { sig, (uint8_t) (vid >> 8), (uint8_t) vid, (uint8_t) (pid >> 8), (uint8_t) pid, (uint8_t) (version >>
-      8), (uint8_t) version }; m_pnpCharacteristic->setValue(pnp, sizeof(pnp));
-      */
-    swC->setValue(swVersion);
-    deviceInfoService->addCharacteristic(addBLECharacteristic(swC));
-    mfC->setValue(hwVendor);
-    deviceInfoService->addCharacteristic(addBLECharacteristic(mfC));
-    if (!hwVersion.empty()) {
-        BLECharacteristic *hwvC =
-            new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_HW_VERSION_STR), BLECharacteristic::PROPERTY_READ);
-        hwvC->setValue(hwVersion);
-        deviceInfoService->addCharacteristic(addBLECharacteristic(hwvC));
-    }
-    // SerialNumberCharacteristic.setValue("FIXME");
-    // deviceInfoService->addCharacteristic(&SerialNumberCharacteristic);
-
-    // m_manufacturerCharacteristic = m_deviceInfoService->createCharacteristic((uint16_t) 0x2a29,
-    // BLECharacteristic::PROPERTY_READ); m_manufacturerCharacteristic->setValue(name);
-
-    /* add these later?
-      ESP_GATT_UUID_SYSTEM_ID
-      */
-
-    // caller must call service->start();
-    return deviceInfoService;
-}
 
 bool _BLEClientConnected = false;
 
@@ -106,6 +59,54 @@ void addWithDesc(BLEService *service, BLECharacteristic *c, const char *descript
     addBLEDescriptor(desc);
 }
 
+/**
+ * Create standard device info service
+ **/
+BLEService *createDeviceInfomationService(BLEServer *server, std::string hwVendor, std::string swVersion,
+                                          std::string hwVersion = "")
+{
+    BLEService *deviceInfoService = server->createService(BLEUUID((uint16_t)ESP_GATT_UUID_DEVICE_INFO_SVC));
+
+    BLECharacteristic *swC = new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_SW_VERSION_STR), BLECharacteristic::PROPERTY_READ);
+    BLECharacteristic *mfC = new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_MANU_NAME), BLECharacteristic::PROPERTY_READ);
+    // BLECharacteristic SerialNumberCharacteristic(BLEUUID((uint16_t) ESP_GATT_UUID_SERIAL_NUMBER_STR),
+    // BLECharacteristic::PROPERTY_READ);
+
+    /*
+           * Mandatory characteristic for device info service?
+
+          BLECharacteristic *m_pnpCharacteristic = m_deviceInfoService->createCharacteristic(ESP_GATT_UUID_PNP_ID,
+      BLECharacteristic::PROPERTY_READ);
+
+      uint8_t sig, uint16_t vid, uint16_t pid, uint16_t version;
+          uint8_t pnp[] = { sig, (uint8_t) (vid >> 8), (uint8_t) vid, (uint8_t) (pid >> 8), (uint8_t) pid, (uint8_t) (version >>
+      8), (uint8_t) version }; m_pnpCharacteristic->setValue(pnp, sizeof(pnp));
+      */
+    swC->setValue(swVersion);
+    deviceInfoService->addCharacteristic(addBLECharacteristic(swC));
+    mfC->setValue(hwVendor);
+    deviceInfoService->addCharacteristic(addBLECharacteristic(mfC));
+    if (!hwVersion.empty()) {
+        BLECharacteristic *hwvC =
+            new BLECharacteristic(BLEUUID((uint16_t)ESP_GATT_UUID_HW_VERSION_STR), BLECharacteristic::PROPERTY_READ);
+        hwvC->setValue(hwVersion);
+        deviceInfoService->addCharacteristic(addBLECharacteristic(hwvC));
+    }
+    // SerialNumberCharacteristic.setValue("FIXME");
+    // deviceInfoService->addCharacteristic(&SerialNumberCharacteristic);
+
+    // m_manufacturerCharacteristic = m_deviceInfoService->createCharacteristic((uint16_t) 0x2a29,
+    // BLECharacteristic::PROPERTY_READ); m_manufacturerCharacteristic->setValue(name);
+
+    /* add these later?
+      ESP_GATT_UUID_SYSTEM_ID
+      */
+
+    // caller must call service->start();
+    return deviceInfoService;
+}
+
+
 static BLECharacteristic *batteryLevelC;
 
 /**
@@ -223,11 +224,15 @@ void deinitBLE()
 
     pServer->getAdvertising()->stop();
 
-    destroyUpdateService();
+    if (pUpdate != NULL) {
+        destroyUpdateService();
+
+        pUpdate->stop(); // we delete them below
+        pUpdate->executeDelete();
+    }
 
-    pUpdate->stop();
     pDevInfo->stop();
-    pUpdate->stop(); // we delete them below
+    pDevInfo->executeDelete();
 
     // First shutdown bluetooth
     BLEDevice::deinit(false);
@@ -235,14 +240,16 @@ void deinitBLE()
     // do not delete this - it is dynamically allocated, but only once - statically in BLEDevice
     // delete pServer->getAdvertising();
 
-    delete pUpdate;
+    if (pUpdate != NULL)
+        delete pUpdate;
     delete pDevInfo;
     delete pServer;
 
     batteryLevelC = NULL; // Don't let anyone generate bogus notifies
 
-    for (int i = 0; i < numChars; i++)
+    for (int i = 0; i < numChars; i++) {
         delete chars[i];
+    }
     numChars = 0;
 
     for (int i = 0; i < numDescs; i++)
@@ -276,15 +283,19 @@ BLEServer *initBLE(StartBluetoothPinScreenCallback startBtPinScreen, StopBluetoo
     // We now let users create the battery service only if they really want (not all devices have a battery)
     // BLEService *pBattery = createBatteryService(pServer);
 
+// #define BLE_SOFTWARE_UPDATE
+#ifdef BLE_SOFTWARE_UPDATE
     pUpdate = createUpdateService(pServer, hwVendor, swVersion,
                                   hwVersion); // We need to advertise this so our android ble scan operation can see it
 
+    pUpdate->start();
+#endif
+
     // It seems only one service can be advertised - so for now don't advertise our updater
     // pServer->getAdvertising()->addServiceUUID(pUpdate->getUUID());
 
     // start all our services (do this after creating all of them)
     pDevInfo->start();
-    pUpdate->start();
 
     // FIXME turn on this restriction only after the device is paired with a phone
     // advert->setScanFilter(false, true); // We let anyone scan for us (FIXME, perhaps only allow that until we are paired with a

src/esp32/MeshBluetoothService.cpp

@@ -41,39 +41,6 @@ class BluetoothPhoneAPI : public PhoneAPI
 
 BluetoothPhoneAPI *bluetoothPhoneAPI;
 
-class ProtobufCharacteristic : public CallbackCharacteristic
-{
-    const pb_msgdesc_t *fields;
-    void *my_struct;
-
-  public:
-    ProtobufCharacteristic(const char *uuid, uint32_t btprops, const pb_msgdesc_t *_fields, void *_my_struct)
-        : CallbackCharacteristic(uuid, btprops), fields(_fields), my_struct(_my_struct)
-    {
-        setCallbacks(this);
-    }
-
-    void onRead(BLECharacteristic *c)
-    {
-        size_t numbytes = pb_encode_to_bytes(trBytes, sizeof(trBytes), fields, my_struct);
-        DEBUG_MSG("pbread from %s returns %d bytes\n", c->getUUID().toString().c_str(), numbytes);
-        c->setValue(trBytes, numbytes);
-    }
-
-    void onWrite(BLECharacteristic *c) { writeToDest(c, my_struct); }
-
-  protected:
-    /// like onWrite, but we provide an different destination to write to, for use by subclasses that
-    /// want to optionally ignore parts of writes.
-    /// returns true for success
-    bool writeToDest(BLECharacteristic *c, void *dest)
-    {
-        // dumpCharacteristic(pCharacteristic);
-        std::string src = c->getValue();
-        DEBUG_MSG("pbwrite to %s of %d bytes\n", c->getUUID().toString().c_str(), src.length());
-        return pb_decode_from_bytes((const uint8_t *)src.c_str(), src.length(), fields, dest);
-    }
-};
 
 class ToRadioCharacteristic : public CallbackCharacteristic
 {
@@ -82,8 +49,6 @@ class ToRadioCharacteristic : public CallbackCharacteristic
 
     void onWrite(BLECharacteristic *c)
     {
-        DEBUG_MSG("Got on write\n");
-
         bluetoothPhoneAPI->handleToRadio(c->getData(), c->getValue().length());
     }
 };
@@ -166,6 +131,7 @@ void stopMeshBluetoothService()
 {
     assert(meshService);
     meshService->stop();
+    meshService->executeDelete();
 }
 
 void destroyMeshBluetoothService()

src/esp32/main-esp32.cpp

@@ -160,6 +160,11 @@ void esp32Setup()
     DEBUG_MSG("Setting random seed %u\n", seed);
     randomSeed(seed); // ESP docs say this is fairly random
 
+    DEBUG_MSG("Total heap: %d\n", ESP.getHeapSize());
+    DEBUG_MSG("Free heap: %d\n", ESP.getFreeHeap());
+    DEBUG_MSG("Total PSRAM: %d\n", ESP.getPsramSize());
+    DEBUG_MSG("Free PSRAM: %d\n", ESP.getFreePsram());
+
 #ifdef AXP192_SLAVE_ADDRESS
     axp192Init();
 #endif

src/main.cpp

@@ -33,6 +33,7 @@
 #include "power.h"
 // #include "rom/rtc.h"
 #include "DSRRouter.h"
+#include "debug.h"
 #include "main.h"
 #include "screen.h"
 #include "sleep.h"
@@ -282,6 +283,8 @@ void loop()
     DEBUG_PORT.loop(); // Send/receive protobufs over the serial port
 #endif
 
+    // heap_caps_check_integrity_all(true); // FIXME - disable this expensive check
+
 #ifndef NO_ESP32
     esp32Loop();
 #endif
@@ -314,6 +317,14 @@ void loop()
         showingBootScreen = false;
     }
 
+#ifdef DEBUG_STACK
+    static uint32_t lastPrint = 0;
+    if (millis() - lastPrint > 10 * 1000L) {
+        lastPrint = millis();
+        meshtastic::printThreadInfo("main");
+    }
+#endif
+
     // Update the screen last, after we've figured out what to show.
     screen.debug()->setNodeNumbersStatus(nodeDB.getNumOnlineNodes(), nodeDB.getNumNodes());
     screen.debug()->setChannelNameStatus(channelSettings.name);

src/mesh/MemoryPool.h

@@ -5,12 +5,80 @@
 
 #include "PointerQueue.h"
 
+template <class T> class Allocator
+{
+
+  public:
+    virtual ~Allocator() {}
+
+    /// Return a queable object which has been prefilled with zeros.  Panic if no buffer is available
+    /// Note: this method is safe to call from regular OR ISR code
+    T *allocZeroed()
+    {
+        T *p = allocZeroed(0);
+
+        assert(p); // FIXME panic instead
+        return p;
+    }
+
+    /// Return a queable object which has been prefilled with zeros - allow timeout to wait for available buffers (you probably
+    /// don't want this version).
+    T *allocZeroed(TickType_t maxWait)
+    {
+        T *p = alloc(maxWait);
+
+        if (p)
+            memset(p, 0, sizeof(T));
+        return p;
+    }
+
+    /// Return a queable object which is a copy of some other object
+    T *allocCopy(const T &src, TickType_t maxWait = portMAX_DELAY)
+    {
+        T *p = alloc(maxWait);
+        assert(p);
+
+        if (p)
+            *p = src;
+        return p;
+    }
+
+    /// Return a buffer for use by others
+    virtual void release(T *p) = 0;
+
+  protected:
+    // Alloc some storage
+    virtual T *alloc(TickType_t maxWait) = 0;
+};
+
+/**
+ * An allocator that just uses regular free/malloc
+ */
+template <class T> class MemoryDynamic : public Allocator<T>
+{
+  public:
+    /// Return a buffer for use by others
+    virtual void release(T *p)
+    {
+        assert(p);
+        free(p);
+    }
+
+  protected:
+    // Alloc some storage
+    virtual T *alloc(TickType_t maxWait)
+    {
+        T *p = (T *)malloc(sizeof(T));
+        assert(p);
+        return p;
+    }
+};
+
 /**
  * A pool based allocator
  *
- * Eventually this routine will even be safe for ISR use...
  */
-template <class T> class MemoryPool
+template <class T> class MemoryPool : public Allocator<T>
 {
     PointerQueue<T> dead;
 
@@ -30,39 +98,8 @@ template <class T> class MemoryPool
 
     ~MemoryPool() { delete[] buf; }
 
-    /// Return a queable object which has been prefilled with zeros.  Panic if no buffer is available
-    /// Note: this method is safe to call from regular OR ISR code
-    T *allocZeroed()
-    {
-        T *p = allocZeroed(0);
-
-        assert(p); // FIXME panic instead
-        return p;
-    }
-
-    /// Return a queable object which has been prefilled with zeros - allow timeout to wait for available buffers (you probably
-    /// don't want this version).
-    T *allocZeroed(TickType_t maxWait)
-    {
-        T *p = dead.dequeuePtr(maxWait);
-
-        if (p)
-            memset(p, 0, sizeof(T));
-        return p;
-    }
-
-    /// Return a queable object which is a copy of some other object
-    T *allocCopy(const T &src, TickType_t maxWait = portMAX_DELAY)
-    {
-        T *p = dead.dequeuePtr(maxWait);
-
-        if (p)
-            *p = src;
-        return p;
-    }
-
     /// Return a buffer for use by others
-    void release(T *p)
+    virtual void release(T *p)
     {
         assert(dead.enqueue(p, 0));
         assert(p >= buf &&
@@ -78,4 +115,9 @@ template <class T> class MemoryPool
                (size_t)(p - buf) <
                    maxElements); // sanity check to make sure a programmer didn't free something that didn't come from this pool
     }
+
+  protected:
+    /// Return a queable object which has been prefilled with zeros - allow timeout to wait for available buffers (you
+    /// probably don't want this version).
+    virtual T *alloc(TickType_t maxWait) { return dead.dequeuePtr(maxWait); }
 };

src/mesh/MeshTypes.h

@@ -29,4 +29,4 @@ typedef uint32_t PacketId; // A packet sequence number
 typedef int ErrorCode;
 
 /// Alloc and free packets to our global, ISR safe pool
-extern MemoryPool<MeshPacket> packetPool;
+extern Allocator<MeshPacket> &packetPool;

src/mesh/NodeDB.cpp

@@ -101,10 +101,12 @@ void NodeDB::resetRadioConfig()
     crypto->setKey(channelSettings.psk.size, channelSettings.psk.bytes);
 
     // temp hack for quicker testing
+
     /*
     radioConfig.preferences.screen_on_secs = 30;
     radioConfig.preferences.wait_bluetooth_secs = 30;
-    radioConfig.preferences.position_broadcast_secs = 15;
+    radioConfig.preferences.position_broadcast_secs = 6 * 60;
+    radioConfig.preferences.ls_secs = 60;
     */
 }
 

src/mesh/PhoneAPI.cpp

@@ -42,8 +42,9 @@ void PhoneAPI::handleToRadio(const uint8_t *buf, size_t bufLength)
     if (pb_decode_from_bytes(buf, bufLength, ToRadio_fields, &toRadioScratch)) {
         switch (toRadioScratch.which_variant) {
         case ToRadio_packet_tag: {
-            // If our phone is sending a position, see if we can use it to set our RTC
             MeshPacket &p = toRadioScratch.variant.packet;
+            DEBUG_MSG("PACKET FROM PHONE: id=%d, to=%x, want_ack=%d, which1=%d, which2=%d, typ=%d, buflen=%d\n", p.id, p.to, p.want_ack, p.which_payload,
+            p.decoded.which_payload, p.decoded.data.typ, bufLength);
             service.handleToRadio(p);
             break;
         }
@@ -91,8 +92,12 @@ void PhoneAPI::handleToRadio(const uint8_t *buf, size_t bufLength)
  */
 size_t PhoneAPI::getFromRadio(uint8_t *buf)
 {
-    if (!available())
+    if (!available()) {
+        DEBUG_MSG("getFromRadio, !available\n");
         return false;
+    } else {
+        DEBUG_MSG("getFromRadio, state=%d\n", state);
+    }
 
     // In case we send a FromRadio packet
     memset(&fromRadioScratch, 0, sizeof(fromRadioScratch));

src/mesh/Router.cpp

@@ -19,11 +19,15 @@
     4 // max number of packets destined to our queue, we dispatch packets quickly so it doesn't need to be big
 
 // I think this is right, one packet for each of the three fifos + one packet being currently assembled for TX or RX
+// And every TX packet might have a retransmission packet or an ack alive at any moment
 #define MAX_PACKETS                                                                                                              \
-    (MAX_RX_TOPHONE + MAX_RX_FROMRADIO + MAX_TX_QUEUE +                                                                          \
+    (MAX_RX_TOPHONE + MAX_RX_FROMRADIO + 2 * MAX_TX_QUEUE +                                                                      \
      2) // max number of packets which can be in flight (either queued from reception or queued for sending)
 
-MemoryPool<MeshPacket> packetPool(MAX_PACKETS);
+static MemoryPool<MeshPacket> staticPool(MAX_PACKETS);
+// static MemoryDynamic<MeshPacket> staticPool;
+
+Allocator<MeshPacket> &packetPool = staticPool;
 
 /**
  * Constructor