Merge branch 'develop' into develop

.eslintrc.yml

@@ -1,29 +1,41 @@
-env:
-  node: true
-
-parserOptions:
-  ecmaVersion: 8
-
-extends:
-  - eslint:recommended
-  - plugin:node/recommended
-
-rules:
-  valid-jsdoc: 0
-  func-style: 0
-  no-use-before-define: 0
-  camelcase: 1
-  no-unused-vars: 2
-  no-alert: 2
-  no-console: [2, { allow: ['warn', 'error'] }]
-  no-underscore-dangle: 0
-
-  strict: [2, 'global']
-  no-var: 2
-  prefer-arrow-callback: 2
-  prefer-const: 2
-  no-inner-declarations: 0
-  object-shorthand: [2, 'consistent-as-needed']
-  newline-per-chained-call: 2
-
-  node/no-deprecated-api: 0
+env:
+  node: true
+
+parserOptions:
+  ecmaVersion: 2018
+
+extends:
+  - eslint:recommended
+  - plugin:mocha/recommended
+  - plugin:node/recommended
+
+plugins:
+  - mocha
+  - node
+
+rules:
+  valid-jsdoc: 0
+  func-style: 0
+  no-use-before-define: 0
+  camelcase: 1
+  no-unused-vars: 2
+  no-alert: 2
+  no-console: [2, { allow: ['warn', 'error'] }]
+  no-underscore-dangle: 0
+  object-shorthand: 0
+
+  strict: [2, 'global']
+  no-var: 2
+  prefer-arrow-callback: 2
+  prefer-const: 2
+  no-inner-declarations: 0
+  newline-per-chained-call: 2
+
+  mocha/no-exclusive-tests: 2
+  mocha/no-hooks-for-single-case: 0
+  mocha/no-mocha-arrows: 0
+  mocha/no-setup-in-describe: 0
+  mocha/no-sibling-hooks: 0
+  mocha/no-skipped-tests: 0
+
+  node/no-deprecated-api: 0

.github/issue_template.md

@@ -2,6 +2,11 @@
 You may report several types of issues. Bug reports, enhancements or questions.
 For bug reports however you are required to provice some information so that the 
 issue can be resolved efficiently. The following template should be filled for bugs.
+
+Before submitting the bug just think twice if you really need to submit the bug
+or you may have some issue in your own code, remember that handling issues is
+time consuming, would you better like that we spend time improving the library
+or on non-issues :).
 -->
 
 ## Description

.github/workflows/node.js.yml

@@ -0,0 +1,34 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [develop]
+  pull_request:
+    branches: [develop]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
+    strategy:
+      matrix:
+        node-version: [10.x, 12.x, 14.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: yarn install --frozen-lockfile --non-interactive
+      - run: yarn prettier -- --list-different
+      - run: yarn test

.gitignore

@@ -2,4 +2,6 @@ node_modules
 tmp
 coverage
 *.rdb
-.vscode
+.vscode
+package-lock.json
+.nyc_output

CHANGELOG.md

@@ -1,32 +1,140 @@
 # Changelog
 
+## v.3.20.1
+
+fix(queue): possible infinite loop when disconnect fixes #1746
+fix(clean-priority): remove job from priority set on clean (#1405)
+fix(sandbox): job update (#1957)
+fix: use async version of process.send for progress and log (#1948)
+fix: promote jobs to the right "list" when paused
+
+
+## v.3.20.0
+
+- feat(job): implement Job#retriedOn (#1868)
+- fix: job default opts fixes #1904
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.19.1...v3.20.0)
+
+## v.3.19.1
+
+- fix(getters): properly zip hmget result
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.19.0...v3.19.1)
+
+## v.3.19.0
+
+- feat: add option to exclude data on getters (#1910)
+- fix: lock ioredis to 4.18.0 to avoid breakage with newer 4.19.0+.
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.18.1...v3.19.0)
+
+## v.3.18.1
+
+- fix(repeat): remove last delayed job.
+- fix(rate-limiter): increment jobCounter only when a job is processed. fixes #1875.
+- fix(sandbox): clear dangling timeout.
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.18.0...v3.18.1)
+
+## v.3.18.0
+
+- feat: make pause forward compatible with bullmq (#1818) (@manast)
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.17.0...v3.18.0)
+
+## v.3.17.0
+
+- feat: better rate limiter (#1816) (@manast)
+- feat(sandbox): kill child workers gracefully (#1802) (@GabrielCastro)
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.16.0...v3.17.0)
+
+## v.3.16.0
+
+- feat(rate-limiter): add grouping support.
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.15.0...v3.16.0)
+
+## v.3.15.0
+
+- feat: add isPaused fixes #1274
+- fix: emit waiting event when adding a priority job (#1134)
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.14.0...v3.15.0)
+
+## v.3.14.0
+
+- feat(queue): add removeJobs function
+- fix: clamp negative job delay values to 0 to prevent thrashing
+- fix: use DEFAULT_JOB_NAME (#1585)
+- fix: remove the lazy client error handler on close (#1605)
+- fix: prevent exceeding the maximum stack call size when emptying large queues (#1660)
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.13.0...v3.14.0)
+
+## v.3.13.0
+
+- feat: add "preventParsingData" job option to prevent data parsing
+- fix: queue.clean clean job logs as well
+- fix: whenCurrentJobsFinished should wait for all jobs
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.12.1...v3.13.0)
+
+## v.3.12.1
+
+- fix: catch errors parsing invalid progress data
+- fix(pause): don't initialize bclient if not waiting for jobs to finish
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.12.0...v3.12.1)
+
+## v.3.12.0
+
+- feat: support async custom backoffs.
+- feat(sandbox): emulate job.progress function.
+
+- fix: queue.pause(true, true) doesn't pause queue.
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.11.0...v3.12.0)
+
+## v.3.11.0
+
+- feat(queue): basic support for bulk adding jobs.
+- feat(job): save data on job instance when updated.
+
+- fix(queue): whenCurrentJobsFinished shouldn't initialize bclient. Fixes #1346.
+- fix(queue): unhandled promise warning in updateDelayTimer.
+- fix(sandbox): if the child process is killed, remove it from the pool.
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.10.0...v3.11.0)
+
 ## v.3.10.0
 
 - fix: remove logs automtically when using autoremove fixes #1330
 - feat: add support for keeping a specified number of jobs when using auto-remove.
 - feat: add support for node 12
-- fix: fix check for supported file types in sandboxed processors #1311 
+- fix: fix check for supported file types in sandboxed processors #1311
 - ci: drop support for node 6
 
- [Changes](https://github.com/OptimalBits/bull/compare/v3.9.1...v3.10.0)
+[Changes](https://github.com/OptimalBits/bull/compare/v3.9.1...v3.10.0)
 
 ## v.3.9.1
- 
- - fix: add log to job wrapper
- 
- [Changes](https://github.com/OptimalBits/bull/compare/v3.9.0...v3.9.1)
+
+- fix: add log to job wrapper
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.9.0...v3.9.1)
 
 ## v.3.9.0
- 
- - feat: add job.log #1165
- 
- [Changes](https://github.com/OptimalBits/bull/compare/v3.8.1...v3.9.0)
+
+- feat: add job.log #1165
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.8.1...v3.9.0)
 
 ## v.3.8.1
- 
- - fix: wait for ready in cleanJobsInSet fixes #1298
- 
- [Changes](https://github.com/OptimalBits/bull/compare/v3.8.0...v3.8.1)
+
+- fix: wait for ready in cleanJobsInSet fixes #1298
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.8.0...v3.8.1)
 
 ## v.3.8.0
 

PATTERNS.md

@@ -66,7 +66,13 @@ The most robust and scalable way to accomplish this is by combining the standard
 Reusing Redis Connections
 -------------------------
 
-A standard queue requires **3 connections** to the Redis server. In some situations you might want to re-use connections—for example on Heroku where the connection count is restricted. You can do this with the `createClient` option in the `Queue` constructor:
+A standard queue requires **3 connections** to the Redis server. In some situations you might want to re-use connections—for example on Heroku where the connection count is restricted. You can do this with the `createClient` option in the `Queue` constructor.
+
+Notes:
+- bclient connections [cannot be re-used](https://github.com/OptimalBits/bull/issues/880), so you should return a new connection each time this is called.
+- client and subscriber connections can be shared and will not be closed when the queue is closed.  When you are shutting down the process, first close the queues, then the shared connections (if they are shared).
+- if you are not sharing connections but still using `createClient` to do some custom connection logic, you may still need to keep a list of all the connections you created so you can manually close them later when the queue shuts down, if you need a graceful shutdown for your process
+- do not set a `keyPrefix` on the connection you create, use bull's built-in prefix feature if you need a key prefix
 
 ```js
 var {REDIS_URL} = process.env
@@ -82,8 +88,10 @@ var opts = {
         return client;
       case 'subscriber':
         return subscriber;
-      default:
+      case 'bclient':
         return new Redis(REDIS_URL);
+      default:
+        throw new Error('Unexpected connection type: ', type);
     }
   }
 }
@@ -95,15 +103,15 @@ var queueQux = new Queue('quxbaz', opts);
 Redis cluster
 -------------
 
-Bull internals requires atomic operations that spans different keys. This fact breaks Redis'
-rules for cluster configurations. However it is still possible to use a cluster environment
+Bull internals require atomic operations that span different keys. This behavior breaks Redis's
+rules for cluster configurations. However, it is still possible to use a cluster environment
 by using the proper bull prefix option as a cluster "hash tag". Hash tags are used to guarantee
 that certain keys are placed in the same hash slot, read more about hash tags in the [redis cluster
-tutorial](https://redis.io/topics/cluster-tutorial).
+tutorial](https://redis.io/topics/cluster-tutorial). A hash tag is defined with brackets. I.e. a key that has a substring inside brackets will use that
+substring to determine in which hash slot the key will be placed.
 
-A hash tag is defined with brackets. I.e. a key that has a substring inside brackets will use that
-substring to determine in which hash slot the key will be placed. So to make bull compatible with
-cluster, just use a queue prefix inside brackets, for example:
+In summary, to make bull compatible with Redis cluster, use a queue prefix inside brackets.
+For example:
 
 ```js
   var queue = new Queue('cluster', {