运行一堆Firebase查询后60秒内Firebase超时云功能

Question

我正在使用Firebase作为群组协作应用程序(如Whatsapp),我正在使用云功能来确定哪些手机通讯录也在使用我的应用程序(再次类似于Whatsapp).云函数运行正常,直到我开始在函数日志中看到以下日志以进行某些调用.

Function execution took 60023 ms, finished with status: 'timeout'

我做了一些调试,发现对于这个特定的用户,他在手机的通讯录上有很多联系人,所以显然需要弄清楚哪些联系人正在使用该应用程序的工作也增加到了需要花费的时间. 60秒 以下是Cloud Function的代码

      // contactsData is an array of contacts on the user's phone
      // Each contact can contain one more phone numbers which are
      // present in the phoneNumbers array. So, essentially, we need
      // to query over all the phone numbers in the user's contact book
      contactsData.forEach((contact) => {
        contact.phoneNumbers.forEach((phoneNumber) => {
          // Find if user with this phoneNumber is using the app
          // Check against mobileNumber and mobileNumberWithCC
          promises.push(ref.child('users').orderByChild("mobileNumber").
            equalTo(phoneNumber.number).once("value").then(usersSnapshot => {
              // usersSnapshot should contain just one entry assuming
              // that the phoneNumber will be unique to the user
              if(!usersSnapshot.exists()) {
                return null
              }
              var user = null
              usersSnapshot.forEach(userSnapshot => {
                user = userSnapshot.val()
              })
              return {
                name: contact.name,
                mobileNumber: phoneNumber.number,
                id: user.id
              }
            }))
          promises.push(ref.child('users').orderByChild("mobileNumberWithCC").
            equalTo(phoneNumber.number).once("value").then(usersSnapshot => {
              // usersSnapshot should contain just one entry assuming
              // that the phoneNumber will be unique to the user
              if(!usersSnapshot.exists()) {
                return null
              }
              var user = null
              usersSnapshot.forEach(userSnapshot => {
                user = userSnapshot.val()
              })
              return {
                name: contact.name,
                mobileNumber: phoneNumber.number,
                id: user.id
              }
            }))
        });
      });
      return Promise.all(promises)
    }).then(allContacts => {
      // allContacts is an array of nulls and contacts using the app
      // Get rid of null and any duplicate entries in the returned array
      currentContacts = arrayCompact(allContacts)

      // Create contactsObj which will the user's contacts that are using the app
      currentContacts.forEach(contact => {
        contactsObj[contact.id] = contact
      })
      // Return the currently present contacts
      return ref.child('userInfos').child(uid).child('contacts').once('value')
    }).then((contactsSnapshot) => {
      if(contactsSnapshot.exists()) {
        contactsSnapshot.forEach((contactSnapshot) => {
          previousContacts.push(contactSnapshot.val())
        })
      }
      // Update the contacts on firease asap after reading the previous contacts
      ref.child('userInfos').child(uid).child('contacts').set(contactsObj)

      // Figure out the new, deleted and renamed contacts
      newContacts = arrayDifferenceWith(currentContacts, previousContacts, 
        (obj1, obj2) => (obj1.id === obj2.id))
      deletedContacts = arrayDifferenceWith(previousContacts, currentContacts,
        (obj1, obj2) => (obj1.id === obj2.id))
      renamedContacts = arrayIntersectionWith(currentContacts, previousContacts,
        (obj1, obj2) => (obj1.id === obj2.id && obj1.name !== obj2.name))
      // Create the deletedContactsObj to store on firebase
      deletedContacts.forEach((deletedContact) => {
        deletedContactsObj[deletedContact.id] = deletedContact
      })
      // Get the deleted contacts
      return ref.child('userInfos').child(uid).child('deletedContacts').once('value')
    }).then((deletedContactsSnapshot) => {
      if(deletedContactsSnapshot.exists()) {
        deletedContactsSnapshot.forEach((deletedContactSnapshot) => {
          previouslyDeletedContacts.push(deletedContactSnapshot.val())
        })
      }
      // Contacts that were previously deleted but now added again
      restoredContacts = arrayIntersectionWith(newContacts, previouslyDeletedContacts,
        (obj1, obj2) => (obj1.id === obj2.id))
      // Removed the restored contacts from the deletedContacts
      restoredContacts.forEach((restoredContact) => {
        deletedContactsObj[restoredContact.id] = null
      })
      // Update groups using any of the deleted, new or renamed contacts
      return ContactsHelper.processContactsData(uid, deletedContacts, newContacts, renamedContacts)
    }).then(() => {
      // Set after retrieving the previously deletedContacts
      return ref.child('userInfos').child(uid).child('deletedContacts').update(deletedContactsObj)
    })

以下是一些示例数据

// This is a sample contactsData
[
  {
    "phoneNumbers": [
      {
        "number": "12324312321",
        "label": "home"
      },
      {
        "number": "2322412132",
        "label": "work"
      }
    ],
    "givenName": "blah5",
    "familyName": "",
    "middleName": ""
  },
  {
    "phoneNumbers": [
      {
        "number": "1231221221",
        "label": "mobile"
      }
    ],
    "givenName": "blah3",
    "familyName": "blah4",
    "middleName": ""
  },
  {
    "phoneNumbers": [
      {
        "number": "1234567890",
        "label": "mobile"
      }
    ],
    "givenName": "blah1",
    "familyName": "blah2",
    "middleName": ""
  }
]



// This is how users are stored on Firebase. This could a lot of users
  "users": {
    "id1" : {
      "countryCode" : "91",
      "id" : "id1",
      "mobileNumber" : "1231211232",
      "mobileNumberWithCC" : "911231211232",
      "name" : "Varun"
    },
    "id2" : {
      "countryCode" : "1",
      "id" : "id2",
      "mobileNumber" : "2342112133",
      "mobileNumberWithCC" : "12342112133",
      "name" : "Ashish"
    },
    "id3" : {
      "countryCode" : "1",
      "id" : "id3",
      "mobileNumber" : "123213421",
      "mobileNumberWithCC" : "1123213421",
      "name" : "Pradeep Singh"
    }
  }

在这种特殊情况下,contactsData包含的1046条目和其中一些条目有两个phoneNumbers.所以,我们假设有一些1500我需要检查的电话号码.我创建查询,比较反对mobileNumber和mobileNumberWithCC数据库中的用户.因此,3000在承诺完成之前,函数将总共进行一些查询,并且我猜测完成所有这些查询需要超过60秒,因此云函数超时.

我的几个问题是:

1. 是否所有这些查询都需要超过60秒？考虑到它在Firebase基础架构中运行,我期待它完成得更快.
2. 有没有办法增加函数的超时限制？我目前正在使用Blaze计划.

我还要感谢上述功能的任何替代实施建议,以减轻问题.谢谢!

Answer 1

如果您无法避免查询这么多数据,则可以使用左侧的"函数"产品更改项目的云控制台中函数的超时.目前,您必须在每次新部署时重置超时.

Answer 2

问题

您遇到的性能问题来自ref.child('users').orderByChild("mobileNumber").equalTo(phon??eNumber.number).once??("value")您forEach()在另一个内部调用的查询 forEach().

为了打破这个查询,你基本上是要求数据库迭代通过的孩子/users,比较关键mobileNumber到phon??eNumber.number,如果他们匹配,返回一个值.但是,你不仅要为这两个调用它,mobileNumber而且mobileNumberWithCC你在每次迭代时调用它forEach().因此,这意味着您正在查看X大量用户Y的电话号码Z数量,联系人数量,因此执行X*Y*Z内部数据库操作.这显然很费力,因此您的查询需要超过60秒才能处理.

潜在的修复

我建议在你的数据库上实现一个名为的索引/phoneNumbers.每个密钥/phoneNumbers都要命名,n###########或者c###########包含与该电话号码关联的用户ID"阵列".

这种结构看起来类似于:

"phoneNumbers": {
  "n1234567890": { // without CC, be warned of overlap
    "userId1": true,
    "userId3": true
  },
  "c011234567890": { // with CC for US
    "userId1": true
  },
  "c611234567890": { // with CC for AU
    "userId3": true
  },
  ...
}

笔记:

为什么格式存储的电话号码n###########和c###########？

这是因为Firebase将数字键视为数组的索引.这对于这个用例没有意义,所以我们在开头添加n/ c来抑制这种行为.

为什么要使用n###########和c###########？

如果所有条目仅使用n前缀,则11位电话号码可能与添加了国家/地区代码的10位电话号码重叠.因此,我们使用n普通电话号码和c包含其国家/地区代码的号码.

为什么你说每个键都/phoneNumbers包含一个用户ID 的"数组"？

这是因为您应该避免在Firebase数据库(以及通常的数组)中使用数字索引数组.假设两个单独的进程想要/phoneNumbers/n1234567890通过删除用户ID 来更新.如果想要删除位置1的ID而另一个想要删除位置2的ID; 他们最终会删除位置1和3的ID.这可以通过将用户ID存储为密钥来克服,这允许您通过ID而不是位置来添加/删除它.

履行

由于您已经在使用云功能,因此实现此类索引相对简单.可以根据用户数据轻松调整此代码以适应任何类型的自动生成的索引.

// Initialize functions and admin.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);

/**
 * Listens to operations on the children of `/users` and updates the `/phoneNumbers` index appropriately.
 */
exports.handleNewUser = functions.database.ref('/users/{userId}')
  .onWrite(event => {
    var deltaSnapshot = event.data,
        userId = event.params.userId,
        tasks = []; // for returned promises

    if (!deltaSnapshot.exists()) {
      // This user has been deleted.
      var previousData = deltaSnapshot.previous.val();
      if (previousData.number) {
        tasks.push(removeUserFromPhoneNumber(userId, previousData.number, false));
      }
      if (previousData.numberWithCC) {
        tasks.push(removeUserFromPhoneNumber(userId, previousData.numberWithCC, true));
      }
      // Handle other cleanup tasks.
      return Promise.all(tasks).then(() => {
        console.log('User "' + userId + '" deleted successfully.');
      });
    }

    var currentData = deltaSnapshot.val();

    if (deltaSnapshot.previous.exists()) {
      // This is an update to existing data.
      var previousData = deltaSnapshot.previous.val();

      if (currentData.number != previousData.number) { // Phone number changed.
        tasks.push(removeUserFromPhoneNumber(userId, previousData.number, false));
        tasks.push(addUserToPhoneNumber(userId, currentData.number, false));
      }
      if (currentData.numberWithCC != previousData.numberWithCC) { // Phone number changed.
        tasks.push(removeUserFromPhoneNumber(userId, previousData.numberWithCC, true));
        tasks.push(addUserToPhoneNumber(userId, currentData.numberWithCC, true));
      }
      // Handle other tasks related to update.
      return Promise.all(tasks).then(() => {
        console.log('User "' + userId + '" updated successfully.');
      });
    }

    // If here, this is a new user.
    tasks.push(addUserToPhoneNumber(userId, currentData.number, false));
    tasks.push(addUserToPhoneNumber(userId, currentData.numberWithCC, true));
    // Handle other tasks related to addition of new user.
    return Promise.all(tasks).then(() => {
      console.log('User "' + userId + '" created successfully.');
    });
  );

/* Phone Number Index Helper Functions */

/**
 * Returns an array of user IDs linked to the specified phone number.
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - a promise returning an array of user IDs, may be empty.
 */
function lookupUsersByPhoneNumber(number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return lookupIdsByIndex('phoneNumbers', (withCountryCode ? 'c' : 'n') + number);
}

/**
 * Adds the user ID under the specified phone number's index.
 * @param {String} userId - the user ID
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - the promise returned by transaction()
 */
function addUserToPhoneNumber(userId, number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return addIdToIndex(userId, 'phoneNumbers', (withCountryCode ? 'c' : 'n') + number)
}

/**
 * Removes the user ID under the specified phone number's index.
 * @param {String} userId - the user ID
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - the promise returned by transaction()
 */
function removeUserFromPhoneNumber(userId, number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return removeIdFromIndex(userId, 'phoneNumbers', (withCountryCode ? 'c' : 'n') + number)
}

/* General Firebase Index CRUD APIs */
/* Credit: @samthecodingman */

/**
 * Returns an array of IDs linked to the specified key in the given index.
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function lookupIdsByIndex(indexName, keyName) {
  // Error out before corrupting data.
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName).once("value")
  .then(snapshot => {
    if (!snapshot.exists()) return []; // Use empty array for 'no data'
    var idsObject = snapshot.val();
    if (idsObject == null) return [];
    return Object.keys(idsObject); // return array of IDs
  });
}

/**
 * Adds the ID to the index under the named key.
 * @param {String} id - the entry ID
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function addIdToIndex(id, indexName, keyName) {
  // Error out before corrupting data.
  if (!id) return Promise.reject(new TypeError('id cannot be falsy.');
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName)
  .transaction(function(idsObject) {
    idsObject = idsObject || {}; // Create data if it doesn't exist.
    if (idsObject.hasOwnProperty(id)) return; // No update needed.
    idsObject[id] = true; // Add ID.
    return idsObject;
  });
}

/**
 * Removes the ID from the index under the named key.
 * @param {String} id - the entry ID
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function removeIdFromIndex(id, indexName, keyName) {
  // Error out before corrupting data.
  if (!id) return Promise.reject(new TypeError('id cannot be falsy.');
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName)
  .transaction(function(idsObject) {
    if (idsObject === null) return; // No data to update.
    if (!idsObject.hasOwnProperty(id)) return; // No update needed.
    delete idsObject[id]; // Remove ID.
    if (Object.keys(idsObject).length === 0) return null; // Delete entire entry.
    return idsObject;
  });
}

handleNewUser上述代码段中的函数不会捕获错误.它只会让Firebase处理它们(默认情况下,FB只会记录错误).我建议你按照自己的意愿实现适当的后备(就像你应该使用任何云功能一样).

关于你问题中的源代码,它将变成类似于:

contactsData.forEach((contact) => {
  contact.phoneNumbers.forEach((phoneNumber) => {
    var tasks = [];
    tasks.push(lookupUsersByPhoneNumber(phoneNumber.number, false)); // Lookup without CC
    tasks.push(lookupUsersByPhoneNumber(phoneNumber.number, true)); // Lookup with CC
    Promise.all(tasks).then(taskResults => {
      var i = 0;
      // Elements of taskResults are arrays of strings from the lookup functions.
      // Flatten and dedupe strings arrays
      var userIds = taskResults.reduce((arr, results) => {
        for (i=0;i<results.length;i++) {
          if (results[i] !== null && ~arr.indexOf(results[i])) {
            arr.push(results[i]); // Add if not already added.
          }              
        }
        return arr;
      }, []);

      // Build 'contacts' array (Doesn't need a database lookup!)
      return userIds.map(uid => ({
        name: contact.name,
        phone: phoneNumber.number,
        id: uid
      }));
    }).then(currentContacts => {
      currentContacts.forEach(contact => {
        contactsObj[contact.id] = contact
      });

      // do original code from question here.
      // I'm not 100% on what it does, so I'll leave it to you.
      // It currently uses an array which is a bad implementation (see notes above). Use PUSH to update the contacts rather than deleting and readding them constantly.
    });
  });
});

有关安全/隐私的说明

/phoneNumbers出于隐私原因,我强烈建议限制对云功能服务工作者的读写访问权限.这可能还需要将部分程序逻辑移动到服务器,具体取决于权限问题.

为此,请更换:

admin.initializeApp(functions.config().firebase);

有:

admin.initializeApp(Object.assign({}, functions.config().firebase, {
  databaseAuthVariableOverride: {
    uid: "cloudfunc-service-worker" // change as desired
  }
});

要启用它,您需要按如下方式配置Firebase数据库规则:

"rules": {
    "phoneNumbers": {
      ".read": "'cloudfunc-service-worker' === auth.uid",
      ".write": "'cloudfunc-service-worker' === auth.uid"
    }
  }