在Firestore中删除包含所有子集和嵌套子集的文档

Question

如何删除包含其所有集合和嵌套子集合的文档？(在功能环境中)

在RTDB中,您可以ref.child('../someNode).setValue(null)完成所需的行为.

我可以想到两种方法可以实现所需的删除行为,这两种方式都有可怕的弊端.

1. 创建一个"超级"功能,它将捕获每个文档并批量删除它们.此函数将变得复杂,易于更改,并且可能需要很长的执行时间.

2. 为每种Document类型添加'onDelete'触发器,并使其删除任何直接子集合.您将在根文档上调用delete,删除调用将沿'tree'向下传播.由于功能执行的巨大负荷,这种情况非常缓慢,规模庞大且成本高昂.

想象一下,你必须删除一个'GROUP'及其所有孩子.对于#1来说它会非常混乱,而对于#2来说会很昂贵(每个文档有1个函数调用)

groups > GROUP > projects > PROJECT > files > FILE > assets > ASSET
                                                   > urls > URL
                                    > members > MEMBER
               > questions > QUESTION > answers > ANSWER > replies > REPLY
                                      > comments > COMMENT
               > resources > RESOURCE > submissions > SUBMISSION
                                      > requests > REQUEST

是否有一种优越/偏好/更清晰的方式来删除文档及其所有嵌套的子集合？

考虑到你可以从控制台做到这一点应该是可能的.

Answer 1

根据 firebase 文档：
https ://firebase.google.com/docs/firestore/solutions/delete-collections
删除带有嵌套子集合的集合可能会在服务器端使用 node-JS 轻松整洁地完成。

const client = require('firebase-tools');
await client.firestore
      .delete(collectionPath, {
        project: process.env.GCLOUD_PROJECT,
        recursive: true,
        yes: true
      }); 

Answer 2

现在有一种使用 NodeJS 删除文档及其所有子集合的简单方法。

这在 Nodejs-firestore 版本 v4.11.0 中可用。

从文档中： recursiveDelete() 递归删除指定级别及其下的所有文档和子集合。

import * as admin from 'firebase-admin'

const ref = admin.firestore().doc('my_document')
admin.firestore().recursiveDelete(ref)

Answer 3

对于那些不想或者不能使用云功能的人，我recursiveDelete在admin sdk中找到了一个功能：

https://googleapis.dev/nodejs/firestore/latest/Firestore.html#recursiveDelete

// Recursively delete a reference and log the references of failures.
const bulkWriter = firestore.bulkWriter();
bulkWriter
  .onWriteError((error) => {
    if (error.failedAttempts < MAX_RETRY_ATTEMPTS) {
      return true;
    } else {
      console.log('Failed write at document: ', error.documentRef.path);
      return false;
    }
  });
await firestore.recursiveDelete(docRef, bulkWriter);

Answer 4

不幸的是,你的分析是现实的,事实上这个用例确实需要很多仪式.根据官方文档,对于深删除不支持在单个镜头中firestore既没有通过客户端库,也没有REST的API,也没有cli工具.

它cli是开源的,它的实现在这里:https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js.他们基本上实现了你在问题中描述的选项1.所以你可以从那里获得一些灵感.

选项1.和2.远非理想情况,为了使您的解决方案100%可靠,您需要保留具有删除任务的持久队列,因为长时间运行过程中的任何错误都会使您的系统处于某种不明确的状态.

我不鼓励使用原始选项2.因为递归云函数调用可能很容易出错 - 例如,达到最大值.限制.

如果链接发生变化,请访问https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js的完整来源:

"use strict";

var clc = require("cli-color");
var ProgressBar = require("progress");

var api = require("../api");
var firestore = require("../gcp/firestore");
var FirebaseError = require("../error");
var logger = require("../logger");
var utils = require("../utils");

/**
 * Construct a new Firestore delete operation.
 *
 * @constructor
 * @param {string} project the Firestore project ID.
 * @param {string} path path to a document or collection.
 * @param {boolean} options.recursive true if the delete should be recursive.
 * @param {boolean} options.shallow true if the delete should be shallow (non-recursive).
 * @param {boolean} options.allCollections true if the delete should universally remove all collections and docs.
 */
function FirestoreDelete(project, path, options) {
  this.project = project;
  this.path = path;
  this.recursive = Boolean(options.recursive);
  this.shallow = Boolean(options.shallow);
  this.allCollections = Boolean(options.allCollections);

  // Remove any leading or trailing slashes from the path
  if (this.path) {
    this.path = this.path.replace(/(^\/+|\/+$)/g, "");
  }

  this.isDocumentPath = this._isDocumentPath(this.path);
  this.isCollectionPath = this._isCollectionPath(this.path);

  this.allDescendants = this.recursive;
  this.parent = "projects/" + project + "/databases/(default)/documents";

  // When --all-collections is passed any other flags or arguments are ignored
  if (!options.allCollections) {
    this._validateOptions();
  }
}

/**
 * Validate all options, throwing an exception for any fatal errors.
 */
FirestoreDelete.prototype._validateOptions = function() {
  if (this.recursive && this.shallow) {
    throw new FirebaseError("Cannot pass recursive and shallow options together.");
  }

  if (this.isCollectionPath && !this.recursive && !this.shallow) {
    throw new FirebaseError("Must pass recursive or shallow option when deleting a collection.");
  }

  var pieces = this.path.split("/");

  if (pieces.length === 0) {
    throw new FirebaseError("Path length must be greater than zero.");
  }

  var hasEmptySegment = pieces.some(function(piece) {
    return piece.length === 0;
  });

  if (hasEmptySegment) {
    throw new FirebaseError("Path must not have any empty segments.");
  }
};

/**
 * Determine if a path points to a document.
 *
 * @param {string} path a path to a Firestore document or collection.
 * @return {boolean} true if the path points to a document, false
 * if it points to a collection.
 */
FirestoreDelete.prototype._isDocumentPath = function(path) {
  if (!path) {
    return false;
  }

  var pieces = path.split("/");
  return pieces.length % 2 === 0;
};

/**
 * Determine if a path points to a collection.
 *
 * @param {string} path a path to a Firestore document or collection.
 * @return {boolean} true if the path points to a collection, false
 * if it points to a document.
 */
FirestoreDelete.prototype._isCollectionPath = function(path) {
  if (!path) {
    return false;
  }

  return !this._isDocumentPath(path);
};

/**
 * Construct a StructuredQuery to find descendant documents of a collection.
 *
 * See:
 * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
 *
 * @param {boolean} allDescendants true if subcollections should be included.
 * @param {number} batchSize maximum number of documents to target (limit).
 * @param {string=} startAfter document name to start after (optional).
 * @return {object} a StructuredQuery.
 */
FirestoreDelete.prototype._collectionDescendantsQuery = function(
  allDescendants,
  batchSize,
  startAfter
) {
  var nullChar = String.fromCharCode(0);

  var startAt = this.parent + "/" + this.path + "/" + nullChar;
  var endAt = this.parent + "/" + this.path + nullChar + "/" + nullChar;

  var where = {
    compositeFilter: {
      op: "AND",
      filters: [
        {
          fieldFilter: {
            field: {
              fieldPath: "__name__",
            },
            op: "GREATER_THAN_OR_EQUAL",
            value: {
              referenceValue: startAt,
            },
          },
        },
        {
          fieldFilter: {
            field: {
              fieldPath: "__name__",
            },
            op: "LESS_THAN",
            value: {
              referenceValue: endAt,
            },
          },
        },
      ],
    },
  };

  var query = {
    structuredQuery: {
      where: where,
      limit: batchSize,
      from: [
        {
          allDescendants: allDescendants,
        },
      ],
      select: {
        fields: [{ fieldPath: "__name__" }],
      },
      orderBy: [{ field: { fieldPath: "__name__" } }],
    },
  };

  if (startAfter) {
    query.structuredQuery.startAt = {
      values: [{ referenceValue: startAfter }],
      before: false,
    };
  }

  return query;
};

/**
 * Construct a StructuredQuery to find descendant documents of a document.
 * The document itself will not be included
 * among the results.
 *
 * See:
 * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
 *
 * @param {boolean} allDescendants true if subcollections should be included.
 * @param {number} batchSize maximum number of documents to target (limit).
 * @param {string=} startAfter document name to start after (optional).
 * @return {object} a StructuredQuery.
 */
FirestoreDelete.prototype._docDescendantsQuery = function(allDescendants, batchSize, startAfter) {
  var query = {
    structuredQuery: {
      limit: batchSize,
      from: [
        {
          allDescendants: allDescendants,
        },
      ],
      select: {
        fields: [{ fieldPath: "__name__" }],
      },
      orderBy: [{ field: { fieldPath: "__name__" } }],
    },
  };

  if (startAfter) {
    query.structuredQuery.startAt = {
      values: [{ referenceValue: startAfter }],
      before: false,
    };
  }

  return query;
};

/**
 * Query for a batch of 'descendants' of a given path.
 *
 * For document format see:
 * https://firebase.google.com/docs/firestore/reference/rest/v1beta1/Document
 *
 * @param {boolean} allDescendants true if subcollections should be included,
 * @param {number} batchSize the maximum size of the batch.
 * @param {string=} startAfter the name of the document to start after (optional).
 * @return {Promise<object[]>} a promise for an array of documents.
 */
FirestoreDelete.prototype._getDescendantBatch = function(allDescendants, batchSize, startAfter) {
  var url;
  var body;
  if (this.isDocumentPath) {
    url = this.parent + "/" + this.path + ":runQuery";
    body = this._docDescendantsQuery(allDescendants, batchSize, startAfter);
  } else {
    url = this.parent + ":runQuery";
    body = this._collectionDescendantsQuery(allDescendants, batchSize, startAfter);
  }

  return api
    .request("POST", "/v1beta1/" + url, {
      auth: true,
      data: body,
      origin: api.firestoreOrigin,
    })
    .then(function(res) {
      // Return the 'document' property for each element in the response,
      // where it exists.
      return res.body
        .filter(function(x) {
          return x.document;
        })
        .map(function(x) {
          return x.document;
        });
    });
};

/**
 * Progress bar shared by the class.
 */
FirestoreDelete.progressBar = new ProgressBar("Deleted :current docs (:rate docs/s)", {
  total: Number.MAX_SAFE_INTEGER,
});

/**
 * Repeatedly query for descendants of a path and delete them in batches
 * until no documents remain.
 *
 * @return {Promise} a promise for the entire operation.
 */
FirestoreDelete.prototype._recursiveBatchDelete = function() {
  var self = this;

  // Tunable deletion parameters
  var readBatchSize = 7500;
  var deleteBatchSize = 250;
  var maxPendingDeletes = 15;
  var maxQueueSize = deleteBatchSize * maxPendingDeletes * 2;

  // All temporary variables for the deletion queue.
  var queue = [];
  var numPendingDeletes = 0;
  var pagesRemaining = true;
  var pageIncoming = false;
  var lastDocName;

  var failures = [];
  var retried = {};

  var queueLoop = function() {
    if (queue.length == 0 && numPendingDeletes == 0 && !pagesRemaining) {
      return true;
    }

    if (failures.length > 0) {
      logger.debug("Found " + failures.length + " failed deletes, failing.");
      return true;
    }

    if (queue.length <= maxQueueSize && pagesRemaining && !pageIncoming) {
      pageIncoming = true;

      self
        ._getDescendantBatch(self.allDescendants, readBatchSize, lastDocName)
        .then(function(docs) {
          pageIncoming = false;

          if (docs.length == 0) {
            pagesRemaining = false;
            return;
          }

          queue = queue.concat(docs);
          lastDocName = docs[docs.length - 1].name;
        })
        .catch(function(e) {
          logger.debug("Failed to fetch page after " + lastDocName, e);
          pageIncoming = false;
        });
    }

    if (numPendingDeletes > maxPendingDeletes) {
      return false;
    }

    if (queue.length == 0) {
      return false;
    }

    var toDelete = [];
    var numToDelete = Math.min(deleteBatchSize, queue.length);

    for (var i = 0; i < numToDelete; i++) {
      toDelete.push(queue.shift());
    }

    numPendingDeletes++;
    firestore
      .deleteDocuments(self.project, toDelete)
      .then(function(numDeleted) {
        FirestoreDelete.progressBar.tick(numDeleted);
        numPendingDeletes--;
      })
      .catch(function(e) {
        // For server errors, retry if the document has not yet been retried.
        if (e.status >= 500 && e.status < 600) {
          logger.debug("Server error deleting doc batch", e);

          // Retry each doc up to one time
          toDelete.forEach(function(doc) {
            if (retried[doc.name]) {
              logger.debug("Failed to delete doc " + doc.name + " multiple times.");
              failures.push(doc.name);
            } else {
              retried[doc.name] = true;
              queue.push(doc);
            }
          });
        } else {
          logger.debug("Fatal error deleting docs ", e);
          failures = failures.concat(toDelete);
        }

        numPendingDeletes--;
      });

    return false;
  };

  return new Promise(function(resolve, reject) {
    var intervalId = setInterval(function() {
      if (queueLoop()) {
        clearInterval(intervalId);

        if (failures.length == 0) {
          resolve();
        } else {
          reject("Failed to delete documents " + failures);
        }
      }
    }, 0);
  });
};

/**
 * Delete everything under a given path. If the path represents
 * a document the document is deleted and then all descendants
 * are deleted.
 *
 * @return {Promise} a promise for the entire operation.
 */
FirestoreDelete.prototype._deletePath = function() {
  var self = this;
  var initialDelete;
  if (this.isDocumentPath) {
    var doc = { name: this.parent + "/" + this.path };
    initialDelete = firestore.deleteDocument(doc).catch(function(err) {
      logger.debug("deletePath:initialDelete:error", err);
      if (self.allDescendants) {
        // On a recursive delete, we are insensitive to
        // failures of the initial delete
        return Promise.resolve();
      }

      // For a shallow delete, this error is fatal.
      return utils.reject("Unable to delete " + clc.cyan(this.path));
    });
  } else {
    initialDelete = Promise.resolve();
  }

  return initialDelete.then(function() {
    return self._recursiveBatchDelete();
  });
};

/**
 * Delete an entire database by finding and deleting each collection.
 *
 * @return {Promise} a promise for all of the operations combined.
 */
FirestoreDelete.prototype.deleteDatabase = function() {
  var self = this;
  return firestore
    .listCollectionIds(this.project)
    .catch(function(err) {
      logger.debug("deleteDatabase:listCollectionIds:error", err);
      return utils.reject("Unable to list collection IDs");
    })
    .then(function(collectionIds) {
      var promises = [];

      logger.info("Deleting the following collections: " + clc.cyan(collectionIds.join(", ")));

      for (var i = 0; i < collectionIds.length; i++) {
        var collectionId = collectionIds[i];
        var deleteOp = new FirestoreDelete(self.project, collectionId, {
          recursive: true,
        });

        promises.push(deleteOp.execute());
      }

      return Promise.all(promises);
    });
};

/**
 * Check if a path has any children. Useful for determining
 * if deleting a path will affect more than one document.
 *
 * @return {Promise<boolean>} a promise that retruns true if the path has
 * children and false otherwise.
 */
FirestoreDelete.prototype.checkHasChildren = function() {
  return this._getDescendantBatch(true, 1).then(function(docs) {
    return docs.length > 0;
  });
};

/**
 * Run the delete operation.
 */
FirestoreDelete.prototype.execute = function() {
  var verifyRecurseSafe;
  if (this.isDocumentPath && !this.recursive && !this.shallow) {
    verifyRecurseSafe = this.checkHasChildren().then(function(multiple) {
      if (multiple) {
        return utils.reject("Document has children, must specify -r or --shallow.", { exit: 1 });
      }
    });
  } else {
    verifyRecurseSafe = Promise.resolve();
  }

  var self = this;
  return verifyRecurseSafe.then(function() {
    return self._deletePath();
  });
};

module.exports = FirestoreDelete;