extensions/browser/content_hash_fetcher.cc - chromium/src - Git at Google

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include "extensions/browser/content_hash_fetcher.h"

 #include <stddef.h>

 #include <algorithm>
 #include <memory>
 #include <vector>

 #include "base/base64.h"
 #include "base/files/file_enumerator.h"
 #include "base/files/file_util.h"
 #include "base/json/json_reader.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/synchronization/lock.h"
 #include "base/task_scheduler/post_task.h"
 #include "base/timer/elapsed_timer.h"
 #include "base/version.h"
 #include "content/public/browser/browser_thread.h"
 #include "crypto/sha2.h"
 #include "extensions/browser/computed_hashes.h"
 #include "extensions/browser/content_hash_tree.h"
 #include "extensions/browser/content_verifier_delegate.h"
 #include "extensions/browser/extension_file_task_runner.h"
 #include "extensions/browser/verified_contents.h"
 #include "extensions/common/constants.h"
 #include "extensions/common/extension.h"
 #include "extensions/common/file_util.h"
 #include "net/base/load_flags.h"
 #include "net/traffic_annotation/network_traffic_annotation.h"
 #include "net/url_request/url_fetcher.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "net/url_request/url_request_status.h"

 namespace {

 typedef std::set<base::FilePath> SortedFilePathSet;

 }  // namespace

 namespace extensions {

 // This class takes care of doing the disk and network I/O work to ensure we
 // have both verified_contents.json files from the webstore and
 // computed_hashes.json files computed over the files in an extension's
 // directory.
 class ContentHashFetcherJob
     : public base::RefCountedThreadSafe<ContentHashFetcherJob>,
       public net::URLFetcherDelegate {
  public:
   using CompletionCallback =
       base::Callback<void(scoped_refptr<ContentHashFetcherJob>)>;
   ContentHashFetcherJob(net::URLRequestContextGetter* request_context,
                         const ContentVerifierKey& key,
                         const std::string& extension_id,
                         const base::FilePath& extension_path,
                         const GURL& fetch_url,
                         bool force,
                         const CompletionCallback& callback);

   void Start();

   // Cancels this job, which will attempt to stop I/O operations sooner than
   // just waiting for the entire job to complete. Safe to call from any thread.
   void Cancel();

   // Checks whether this job has been cancelled. Safe to call from any thread.
   bool IsCancelled();

   // Returns whether this job was successful (we have both verified contents
   // and computed hashes). Even if the job was a success, there might have been
   // files that were found to have contents not matching expectations; these
   // are available by calling hash_mismatch_unix_paths().
   bool success() { return success_; }

   bool force() { return force_; }

   const std::string& extension_id() { return extension_id_; }

   // Returns the set of paths (with unix style '/' separators) that had a hash
   // mismatch.
   const std::set<base::FilePath>& hash_mismatch_unix_paths() {
     return hash_mismatch_unix_paths_;
   }

  private:
   friend class base::RefCountedThreadSafe<ContentHashFetcherJob>;
   ~ContentHashFetcherJob() override;

   // Tries to load a verified_contents.json file at |path|. On successfully
   // reading and validing the file, the verified_contents_ member variable will
   // be set and this function will return true. If the file does not exist, or
   // exists but is invalid, it will return false. Also, any invalid
   // file will be removed from disk and
   bool LoadVerifiedContents(const base::FilePath& path);

   // Callback for when we're done doing file I/O to see if we already have
   // a verified contents file. If we don't, this will kick off a network
   // request to get one.
   void DoneCheckingForVerifiedContents(bool found);

   // URLFetcherDelegate interface
   void OnURLFetchComplete(const net::URLFetcher* source) override;

   // Callback for when we're done ensuring we have verified contents, and are
   // ready to move on to MaybeCreateHashes.
   void DoneFetchingVerifiedContents(bool success);

   // Callback for the job to write the verified contents to the filesystem.
   void OnVerifiedContentsWritten(size_t expected_size, int write_result);

   // The verified contents file from the webstore only contains the treehash
   // root hash, but for performance we want to cache the individual block level
   // hashes. This function will create that cache with block-level hashes for
   // each file in the extension if needed (the treehash root hash for each of
   // these should equal what is in the verified contents file from the
   // webstore).
   void MaybeCreateHashes();

   // Computes hashes for all files in |extension_path_|, and uses a
   // ComputedHashes::Writer to write that information into
   // |hashes_file|. Returns true on success.
   bool CreateHashes(const base::FilePath& hashes_file);

   // Will call the callback, if we haven't been cancelled.
   void DispatchCallback();

   net::URLRequestContextGetter* request_context_;
   std::string extension_id_;
   base::FilePath extension_path_;

   // The url we'll need to use to fetch a verified_contents.json file.
   GURL fetch_url_;

   bool force_;

   CompletionCallback callback_;
   content::BrowserThread::ID creation_thread_;

   // Used for fetching content signatures.
   // Created and destroyed on |creation_thread_|.
   std::unique_ptr<net::URLFetcher> url_fetcher_;

   // The key used to validate verified_contents.json.
   ContentVerifierKey key_;

   // The parsed contents of the verified_contents.json file, either read from
   // disk or fetched from the network and then written to disk.
   std::unique_ptr<VerifiedContents> verified_contents_;

   // Whether this job succeeded.
   bool success_;

   // Paths that were found to have a mismatching hash.
   std::set<base::FilePath> hash_mismatch_unix_paths_;

   // The block size to use for hashing.
   int block_size_;

   // Note: this may be accessed from multiple threads, so all access should
   // be protected by |cancelled_lock_|.
   bool cancelled_;

   // A lock for synchronizing access to |cancelled_|.
   base::Lock cancelled_lock_;

   DISALLOW_COPY_AND_ASSIGN(ContentHashFetcherJob);
 };

 ContentHashFetcherJob::ContentHashFetcherJob(
     net::URLRequestContextGetter* request_context,
     const ContentVerifierKey& key,
     const std::string& extension_id,
     const base::FilePath& extension_path,
     const GURL& fetch_url,
     bool force,
     const CompletionCallback& callback)
     : request_context_(request_context),
       extension_id_(extension_id),
       extension_path_(extension_path),
       fetch_url_(fetch_url),
       force_(force),
       callback_(callback),
       key_(key),
       success_(false),
       // TODO(asargent) - use the value from verified_contents.json for each
       // file, instead of using a constant.
       block_size_(4096),
       cancelled_(false) {
   bool got_id =
       content::BrowserThread::GetCurrentThreadIdentifier(&creation_thread_);
   DCHECK(got_id);
 }

 void ContentHashFetcherJob::Start() {
   base::FilePath verified_contents_path =
       file_util::GetVerifiedContentsPath(extension_path_);
   base::PostTaskWithTraitsAndReplyWithResult(
       FROM_HERE, {base::MayBlock(), base::TaskPriority::USER_VISIBLE},
       base::Bind(&ContentHashFetcherJob::LoadVerifiedContents, this,
                  verified_contents_path),
       base::Bind(&ContentHashFetcherJob::DoneCheckingForVerifiedContents,
                  this));
 }

 void ContentHashFetcherJob::Cancel() {
   base::AutoLock autolock(cancelled_lock_);
   cancelled_ = true;
 }

 bool ContentHashFetcherJob::IsCancelled() {
   base::AutoLock autolock(cancelled_lock_);
   bool result = cancelled_;
   return result;
 }

 ContentHashFetcherJob::~ContentHashFetcherJob() {
   // Destroy |url_fetcher_| on correct thread.
   // It was possible for it to be deleted on blocking pool from
   // MaybeCreateHashes(). See https://crbug.com/702300 for details.
   if (url_fetcher_ && !content::BrowserThread::CurrentlyOn(creation_thread_)) {
     content::BrowserThread::DeleteSoon(creation_thread_, FROM_HERE,
                                        url_fetcher_.release());
   }
 }

 bool ContentHashFetcherJob::LoadVerifiedContents(const base::FilePath& path) {
   if (!base::PathExists(path))
     return false;
   verified_contents_.reset(new VerifiedContents(key_.data, key_.size));
   if (!verified_contents_->InitFrom(path)) {
     verified_contents_.reset();
     if (!base::DeleteFile(path, false))
       LOG(WARNING) << "Failed to delete " << path.value();
     return false;
   }
   return true;
 }

 void ContentHashFetcherJob::DoneCheckingForVerifiedContents(bool found) {
   if (IsCancelled())
     return;
   if (found) {
     VLOG(1) << "Found verified contents for " << extension_id_;
     DoneFetchingVerifiedContents(true);
   } else {
     VLOG(1) << "Missing verified contents for " << extension_id_
             << ", fetching...";
     net::NetworkTrafficAnnotationTag traffic_annotation =
         net::DefineNetworkTrafficAnnotation("content_hash_verification_job", R"(
           semantics {
             sender: "Web Store Content Verification"
             description:
               "The request sent to retrieve the file required for content "
               "verification for an extension from the Web Store."
             trigger:
               "An extension from the Web Store is missing the "
               "verified_contents.json file required for extension content "
               "verification."
             data: "The extension id and extension version."
             destination: GOOGLE_OWNED_SERVICE
           }
           policy {
             cookies_allowed: NO
             setting:
               "This feature cannot be directly disabled; it is enabled if any "
               "extension from the webstore is installed in the browser."
             policy_exception_justification:
               "Not implemented, not required. If the user has extensions from "
               "the Web Store, this feature is required to ensure the "
               "extensions match what is distributed by the store."
           })");
     url_fetcher_ = net::URLFetcher::Create(fetch_url_, net::URLFetcher::GET,
                                            this, traffic_annotation);
     url_fetcher_->SetRequestContext(request_context_);
     url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
                                net::LOAD_DO_NOT_SAVE_COOKIES |
                                net::LOAD_DISABLE_CACHE);
     url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
     url_fetcher_->Start();
   }
 }

 // Helper function to let us pass ownership of a string via base::Bind with the
 // contents to be written into a file. Also ensures that the directory for
 // |path| exists, creating it if needed.
 static int WriteFileHelper(const base::FilePath& path,
                            std::unique_ptr<std::string> content) {
   base::FilePath dir = path.DirName();
   if (!base::CreateDirectoryAndGetError(dir, nullptr))
     return -1;
   return base::WriteFile(path, content->data(), content->size());
 }

 void ContentHashFetcherJob::OnURLFetchComplete(const net::URLFetcher* source) {
   VLOG(1) << "URLFetchComplete for " << extension_id_
           << " is_success:" << url_fetcher_->GetStatus().is_success() << " "
           << fetch_url_.possibly_invalid_spec();
   // Delete |url_fetcher_| once we no longer need it.
   std::unique_ptr<net::URLFetcher> url_fetcher = std::move(url_fetcher_);

   if (IsCancelled())
     return;
   std::unique_ptr<std::string> response(new std::string);
   if (!url_fetcher->GetStatus().is_success() ||
       !url_fetcher->GetResponseAsString(response.get())) {
     DoneFetchingVerifiedContents(false);
     return;
   }

   // Parse the response to make sure it is valid json (on staging sometimes it
   // can be a login redirect html, xml file, etc. if you aren't logged in with
   // the right cookies).  TODO(asargent) - It would be a nice enhancement to
   // move to parsing this in a sandboxed helper (crbug.com/372878).
   std::unique_ptr<base::Value> parsed(base::JSONReader::Read(*response));
   if (parsed) {
     VLOG(1) << "JSON parsed ok for " << extension_id_;

     parsed.reset();  // no longer needed
     base::FilePath destination =
         file_util::GetVerifiedContentsPath(extension_path_);
     size_t size = response->size();
     base::PostTaskWithTraitsAndReplyWithResult(
         FROM_HERE, {base::MayBlock(), base::TaskPriority::USER_VISIBLE},
         base::Bind(&WriteFileHelper, destination, base::Passed(&response)),
         base::Bind(&ContentHashFetcherJob::OnVerifiedContentsWritten, this,
                    size));
   } else {
     DoneFetchingVerifiedContents(false);
   }
 }

 void ContentHashFetcherJob::OnVerifiedContentsWritten(size_t expected_size,
                                                       int write_result) {
   bool success =
       (write_result >= 0 && static_cast<size_t>(write_result) == expected_size);
   DoneFetchingVerifiedContents(success);
 }

 void ContentHashFetcherJob::DoneFetchingVerifiedContents(bool success) {
   if (IsCancelled())
     return;

   if (!success) {
     DispatchCallback();
     return;
   }

   GetExtensionFileTaskRunner()->PostTask(
       FROM_HERE, base::Bind(&ContentHashFetcherJob::MaybeCreateHashes, this));
 }

 void ContentHashFetcherJob::MaybeCreateHashes() {
   if (IsCancelled())
     return;
   base::FilePath hashes_file =
       file_util::GetComputedHashesPath(extension_path_);

   if (!force_ && base::PathExists(hashes_file)) {
     success_ = true;
   } else {
     if (force_)
       base::DeleteFile(hashes_file, false /* recursive */);
     success_ = CreateHashes(hashes_file);
   }

   content::BrowserThread::PostTask(
       creation_thread_,
       FROM_HERE,
       base::Bind(&ContentHashFetcherJob::DispatchCallback, this));
 }

 bool ContentHashFetcherJob::CreateHashes(const base::FilePath& hashes_file) {
   base::ElapsedTimer timer;
   if (IsCancelled())
     return false;
   // Make sure the directory exists.
   if (!base::CreateDirectoryAndGetError(hashes_file.DirName(), NULL))
     return false;

   if (!verified_contents_.get()) {
     base::FilePath verified_contents_path =
         file_util::GetVerifiedContentsPath(extension_path_);
     verified_contents_.reset(new VerifiedContents(key_.data, key_.size));
     if (!verified_contents_->InitFrom(verified_contents_path)) {
       verified_contents_.reset();
       return false;
     }
   }

   base::FileEnumerator enumerator(extension_path_,
                                   true, /* recursive */
                                   base::FileEnumerator::FILES);
   // First discover all the file paths and put them in a sorted set.
   SortedFilePathSet paths;
   for (;;) {
     if (IsCancelled())
       return false;

     base::FilePath full_path = enumerator.Next();
     if (full_path.empty())
       break;
     paths.insert(full_path);
   }

   // Now iterate over all the paths in sorted order and compute the block hashes
   // for each one.
   ComputedHashes::Writer writer;
   for (SortedFilePathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
     if (IsCancelled())
       return false;
     const base::FilePath& full_path = *i;
     base::FilePath relative_unix_path;
     extension_path_.AppendRelativePath(full_path, &relative_unix_path);
     relative_unix_path = relative_unix_path.NormalizePathSeparatorsTo('/');

     if (!verified_contents_->HasTreeHashRoot(relative_unix_path))
       continue;

     std::string contents;
     if (!base::ReadFileToString(full_path, &contents)) {
       LOG(ERROR) << "Could not read " << full_path.MaybeAsASCII();
       continue;
     }

     // Iterate through taking the hash of each block of size (block_size_) of
     // the file.
     std::vector<std::string> hashes;
     ComputedHashes::ComputeHashesForContent(contents, block_size_, &hashes);
     std::string root =
         ComputeTreeHashRoot(hashes, block_size_ / crypto::kSHA256Length);
     if (!verified_contents_->TreeHashRootEquals(relative_unix_path, root)) {
       VLOG(1) << "content mismatch for " << relative_unix_path.AsUTF8Unsafe();
       hash_mismatch_unix_paths_.insert(relative_unix_path);
       continue;
     }

     writer.AddHashes(relative_unix_path, block_size_, hashes);
   }
   bool result = writer.WriteToFile(hashes_file);
   UMA_HISTOGRAM_TIMES("ExtensionContentHashFetcher.CreateHashesTime",
                       timer.Elapsed());
   return result;
 }

 void ContentHashFetcherJob::DispatchCallback() {
   {
     base::AutoLock autolock(cancelled_lock_);
     if (cancelled_)
       return;
   }
   callback_.Run(make_scoped_refptr(this));
 }

 // ----

 ContentHashFetcher::ContentHashFetcher(
     net::URLRequestContextGetter* context_getter,
     ContentVerifierDelegate* delegate,
     const FetchCallback& callback)
     : context_getter_(context_getter),
       delegate_(delegate),
       fetch_callback_(callback),
       weak_ptr_factory_(this) {}

 ContentHashFetcher::~ContentHashFetcher() {
   for (JobMap::iterator i = jobs_.begin(); i != jobs_.end(); ++i) {
     i->second->Cancel();
   }
 }

 void ContentHashFetcher::DoFetch(const Extension* extension, bool force) {
   DCHECK(extension);

   IdAndVersion key(extension->id(), extension->version()->GetString());
   JobMap::iterator found = jobs_.find(key);
   if (found != jobs_.end()) {
     if (!force || found->second->force()) {
       // Just let the existing job keep running.
       return;
     } else {
       // Kill the existing non-force job, so we can start a new one below.
       found->second->Cancel();
       jobs_.erase(found);
     }
   }

   // TODO(asargent) - we should do something here to remember recent attempts
   // to fetch signatures by extension id, and use exponential backoff to avoid
   // hammering the server when we aren't successful in getting them.
   // crbug.com/373397

   DCHECK(extension->version());
   GURL url =
       delegate_->GetSignatureFetchUrl(extension->id(), *extension->version());
   ContentHashFetcherJob* job =
       new ContentHashFetcherJob(context_getter_, delegate_->GetPublicKey(),
                                 extension->id(), extension->path(), url, force,
                                 base::Bind(&ContentHashFetcher::JobFinished,
                                            weak_ptr_factory_.GetWeakPtr()));
   jobs_.insert(std::make_pair(key, job));
   job->Start();
 }

 void ContentHashFetcher::ExtensionLoaded(const Extension* extension) {
   CHECK(extension);
   DoFetch(extension, false);
 }

 void ContentHashFetcher::ExtensionUnloaded(const Extension* extension) {
   CHECK(extension);
   IdAndVersion key(extension->id(), extension->version()->GetString());
   JobMap::iterator found = jobs_.find(key);
   if (found != jobs_.end()) {
     found->second->Cancel();
     jobs_.erase(found);
   }
 }

 void ContentHashFetcher::JobFinished(scoped_refptr<ContentHashFetcherJob> job) {
   if (!job->IsCancelled()) {
     // Note: Run can result in ContentHashFetcher::ExtensionUnloaded.
     //
     // TODO(lazyboy): Add a unit test to cover the case where Run can result in
     // ContentHashFetcher::ExtensionUnloaded, once https://crbug.com/702300 is
     // fixed.
     fetch_callback_.Run(job->extension_id(), job->success(), job->force(),
                         job->hash_mismatch_unix_paths());
   }

   for (JobMap::iterator i = jobs_.begin(); i != jobs_.end(); ++i) {
     if (i->second.get() == job.get()) {
       jobs_.erase(i);
       break;
     }
   }
 }

 }  // namespace extensions
	// Copyright 2014 The Chromium Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include "extensions/browser/content_hash_fetcher.h"

	#include <stddef.h>

	#include <algorithm>
	#include <memory>
	#include <vector>

	#include "base/base64.h"
	#include "base/files/file_enumerator.h"
	#include "base/files/file_util.h"
	#include "base/json/json_reader.h"
	#include "base/macros.h"
	#include "base/memory/ref_counted.h"
	#include "base/metrics/histogram_macros.h"
	#include "base/synchronization/lock.h"
	#include "base/task_scheduler/post_task.h"
	#include "base/timer/elapsed_timer.h"
	#include "base/version.h"
	#include "content/public/browser/browser_thread.h"
	#include "crypto/sha2.h"
	#include "extensions/browser/computed_hashes.h"
	#include "extensions/browser/content_hash_tree.h"
	#include "extensions/browser/content_verifier_delegate.h"
	#include "extensions/browser/extension_file_task_runner.h"
	#include "extensions/browser/verified_contents.h"
	#include "extensions/common/constants.h"
	#include "extensions/common/extension.h"
	#include "extensions/common/file_util.h"
	#include "net/base/load_flags.h"
	#include "net/traffic_annotation/network_traffic_annotation.h"
	#include "net/url_request/url_fetcher.h"
	#include "net/url_request/url_fetcher_delegate.h"
	#include "net/url_request/url_request_status.h"

	namespace {

	typedef std::set<base::FilePath> SortedFilePathSet;

	} // namespace

	namespace extensions {

	// This class takes care of doing the disk and network I/O work to ensure we
	// have both verified_contents.json files from the webstore and
	// computed_hashes.json files computed over the files in an extension's
	// directory.
	class ContentHashFetcherJob
	: public base::RefCountedThreadSafe<ContentHashFetcherJob>,
	public net::URLFetcherDelegate {
	public:
	using CompletionCallback =
	base::Callback<void(scoped_refptr<ContentHashFetcherJob>)>;
	ContentHashFetcherJob(net::URLRequestContextGetter* request_context,
	const ContentVerifierKey& key,
	const std::string& extension_id,
	const base::FilePath& extension_path,
	const GURL& fetch_url,
	bool force,
	const CompletionCallback& callback);

	void Start();

	// Cancels this job, which will attempt to stop I/O operations sooner than
	// just waiting for the entire job to complete. Safe to call from any thread.
	void Cancel();

	// Checks whether this job has been cancelled. Safe to call from any thread.
	bool IsCancelled();

	// Returns whether this job was successful (we have both verified contents
	// and computed hashes). Even if the job was a success, there might have been
	// files that were found to have contents not matching expectations; these
	// are available by calling hash_mismatch_unix_paths().
	bool success() { return success_; }

	bool force() { return force_; }

	const std::string& extension_id() { return extension_id_; }

	// Returns the set of paths (with unix style '/' separators) that had a hash
	// mismatch.
	const std::set<base::FilePath>& hash_mismatch_unix_paths() {
	return hash_mismatch_unix_paths_;
	}

	private:
	friend class base::RefCountedThreadSafe<ContentHashFetcherJob>;
	~ContentHashFetcherJob() override;

	// Tries to load a verified_contents.json file at \|path\|. On successfully
	// reading and validing the file, the verified_contents_ member variable will
	// be set and this function will return true. If the file does not exist, or
	// exists but is invalid, it will return false. Also, any invalid
	// file will be removed from disk and
	bool LoadVerifiedContents(const base::FilePath& path);

	// Callback for when we're done doing file I/O to see if we already have
	// a verified contents file. If we don't, this will kick off a network
	// request to get one.
	void DoneCheckingForVerifiedContents(bool found);

	// URLFetcherDelegate interface
	void OnURLFetchComplete(const net::URLFetcher* source) override;

	// Callback for when we're done ensuring we have verified contents, and are
	// ready to move on to MaybeCreateHashes.
	void DoneFetchingVerifiedContents(bool success);

	// Callback for the job to write the verified contents to the filesystem.
	void OnVerifiedContentsWritten(size_t expected_size, int write_result);

	// The verified contents file from the webstore only contains the treehash
	// root hash, but for performance we want to cache the individual block level
	// hashes. This function will create that cache with block-level hashes for
	// each file in the extension if needed (the treehash root hash for each of
	// these should equal what is in the verified contents file from the
	// webstore).
	void MaybeCreateHashes();

	// Computes hashes for all files in \|extension_path_\|, and uses a
	// ComputedHashes::Writer to write that information into
	// \|hashes_file\|. Returns true on success.
	bool CreateHashes(const base::FilePath& hashes_file);

	// Will call the callback, if we haven't been cancelled.
	void DispatchCallback();

	net::URLRequestContextGetter* request_context_;
	std::string extension_id_;
	base::FilePath extension_path_;

	// The url we'll need to use to fetch a verified_contents.json file.
	GURL fetch_url_;

	bool force_;

	CompletionCallback callback_;
	content::BrowserThread::ID creation_thread_;

	// Used for fetching content signatures.
	// Created and destroyed on \|creation_thread_\|.
	std::unique_ptr<net::URLFetcher> url_fetcher_;

	// The key used to validate verified_contents.json.
	ContentVerifierKey key_;

	// The parsed contents of the verified_contents.json file, either read from
	// disk or fetched from the network and then written to disk.
	std::unique_ptr<VerifiedContents> verified_contents_;

	// Whether this job succeeded.
	bool success_;

	// Paths that were found to have a mismatching hash.
	std::set<base::FilePath> hash_mismatch_unix_paths_;

	// The block size to use for hashing.
	int block_size_;

	// Note: this may be accessed from multiple threads, so all access should
	// be protected by \|cancelled_lock_\|.
	bool cancelled_;

	// A lock for synchronizing access to \|cancelled_\|.
	base::Lock cancelled_lock_;

	DISALLOW_COPY_AND_ASSIGN(ContentHashFetcherJob);
	};

	ContentHashFetcherJob::ContentHashFetcherJob(
	net::URLRequestContextGetter* request_context,
	const ContentVerifierKey& key,
	const std::string& extension_id,
	const base::FilePath& extension_path,
	const GURL& fetch_url,
	bool force,
	const CompletionCallback& callback)
	: request_context_(request_context),
	extension_id_(extension_id),
	extension_path_(extension_path),
	fetch_url_(fetch_url),
	force_(force),
	callback_(callback),
	key_(key),
	success_(false),
	// TODO(asargent) - use the value from verified_contents.json for each
	// file, instead of using a constant.
	block_size_(4096),
	cancelled_(false) {
	bool got_id =
	content::BrowserThread::GetCurrentThreadIdentifier(&creation_thread_);
	DCHECK(got_id);
	}

	void ContentHashFetcherJob::Start() {
	base::FilePath verified_contents_path =
	file_util::GetVerifiedContentsPath(extension_path_);
	base::PostTaskWithTraitsAndReplyWithResult(
	FROM_HERE, {base::MayBlock(), base::TaskPriority::USER_VISIBLE},
	base::Bind(&ContentHashFetcherJob::LoadVerifiedContents, this,
	verified_contents_path),
	base::Bind(&ContentHashFetcherJob::DoneCheckingForVerifiedContents,
	this));
	}

	void ContentHashFetcherJob::Cancel() {
	base::AutoLock autolock(cancelled_lock_);
	cancelled_ = true;
	}

	bool ContentHashFetcherJob::IsCancelled() {
	base::AutoLock autolock(cancelled_lock_);
	bool result = cancelled_;
	return result;
	}

	ContentHashFetcherJob::~ContentHashFetcherJob() {
	// Destroy \|url_fetcher_\| on correct thread.
	// It was possible for it to be deleted on blocking pool from
	// MaybeCreateHashes(). See https://crbug.com/702300 for details.
	if (url_fetcher_ && !content::BrowserThread::CurrentlyOn(creation_thread_)) {
	content::BrowserThread::DeleteSoon(creation_thread_, FROM_HERE,
	url_fetcher_.release());
	}
	}

	bool ContentHashFetcherJob::LoadVerifiedContents(const base::FilePath& path) {
	if (!base::PathExists(path))
	return false;
	verified_contents_.reset(new VerifiedContents(key_.data, key_.size));
	if (!verified_contents_->InitFrom(path)) {
	verified_contents_.reset();
	if (!base::DeleteFile(path, false))
	LOG(WARNING) << "Failed to delete " << path.value();
	return false;
	}
	return true;
	}

	void ContentHashFetcherJob::DoneCheckingForVerifiedContents(bool found) {
	if (IsCancelled())
	return;
	if (found) {
	VLOG(1) << "Found verified contents for " << extension_id_;
	DoneFetchingVerifiedContents(true);
	} else {
	VLOG(1) << "Missing verified contents for " << extension_id_
	<< ", fetching...";
	net::NetworkTrafficAnnotationTag traffic_annotation =
	net::DefineNetworkTrafficAnnotation("content_hash_verification_job", R"(
	semantics {
	sender: "Web Store Content Verification"
	description:
	"The request sent to retrieve the file required for content "
	"verification for an extension from the Web Store."
	trigger:
	"An extension from the Web Store is missing the "
	"verified_contents.json file required for extension content "
	"verification."
	data: "The extension id and extension version."
	destination: GOOGLE_OWNED_SERVICE
	}
	policy {
	cookies_allowed: NO
	setting:
	"This feature cannot be directly disabled; it is enabled if any "
	"extension from the webstore is installed in the browser."
	policy_exception_justification:
	"Not implemented, not required. If the user has extensions from "
	"the Web Store, this feature is required to ensure the "
	"extensions match what is distributed by the store."
	})");
	url_fetcher_ = net::URLFetcher::Create(fetch_url_, net::URLFetcher::GET,
	this, traffic_annotation);
	url_fetcher_->SetRequestContext(request_context_);
	url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES \|
	net::LOAD_DO_NOT_SAVE_COOKIES \|
	net::LOAD_DISABLE_CACHE);
	url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
	url_fetcher_->Start();
	}
	}

	// Helper function to let us pass ownership of a string via base::Bind with the
	// contents to be written into a file. Also ensures that the directory for
	// \|path\| exists, creating it if needed.
	static int WriteFileHelper(const base::FilePath& path,
	std::unique_ptr<std::string> content) {
	base::FilePath dir = path.DirName();
	if (!base::CreateDirectoryAndGetError(dir, nullptr))
	return -1;
	return base::WriteFile(path, content->data(), content->size());
	}

	void ContentHashFetcherJob::OnURLFetchComplete(const net::URLFetcher* source) {
	VLOG(1) << "URLFetchComplete for " << extension_id_
	<< " is_success:" << url_fetcher_->GetStatus().is_success() << " "
	<< fetch_url_.possibly_invalid_spec();
	// Delete \|url_fetcher_\| once we no longer need it.
	std::unique_ptr<net::URLFetcher> url_fetcher = std::move(url_fetcher_);

	if (IsCancelled())
	return;
	std::unique_ptr<std::string> response(new std::string);
	if (!url_fetcher->GetStatus().is_success() \|\|
	!url_fetcher->GetResponseAsString(response.get())) {
	DoneFetchingVerifiedContents(false);
	return;
	}

	// Parse the response to make sure it is valid json (on staging sometimes it
	// can be a login redirect html, xml file, etc. if you aren't logged in with
	// the right cookies). TODO(asargent) - It would be a nice enhancement to
	// move to parsing this in a sandboxed helper (crbug.com/372878).
	std::unique_ptr<base::Value> parsed(base::JSONReader::Read(*response));
	if (parsed) {
	VLOG(1) << "JSON parsed ok for " << extension_id_;

	parsed.reset(); // no longer needed
	base::FilePath destination =
	file_util::GetVerifiedContentsPath(extension_path_);
	size_t size = response->size();
	base::PostTaskWithTraitsAndReplyWithResult(
	FROM_HERE, {base::MayBlock(), base::TaskPriority::USER_VISIBLE},
	base::Bind(&WriteFileHelper, destination, base::Passed(&response)),
	base::Bind(&ContentHashFetcherJob::OnVerifiedContentsWritten, this,
	size));
	} else {
	DoneFetchingVerifiedContents(false);
	}
	}

	void ContentHashFetcherJob::OnVerifiedContentsWritten(size_t expected_size,
	int write_result) {
	bool success =
	(write_result >= 0 && static_cast<size_t>(write_result) == expected_size);
	DoneFetchingVerifiedContents(success);
	}

	void ContentHashFetcherJob::DoneFetchingVerifiedContents(bool success) {
	if (IsCancelled())
	return;

	if (!success) {
	DispatchCallback();
	return;
	}

	GetExtensionFileTaskRunner()->PostTask(
	FROM_HERE, base::Bind(&ContentHashFetcherJob::MaybeCreateHashes, this));
	}

	void ContentHashFetcherJob::MaybeCreateHashes() {
	if (IsCancelled())
	return;
	base::FilePath hashes_file =
	file_util::GetComputedHashesPath(extension_path_);

	if (!force_ && base::PathExists(hashes_file)) {
	success_ = true;
	} else {
	if (force_)
	base::DeleteFile(hashes_file, false /* recursive */);
	success_ = CreateHashes(hashes_file);
	}

	content::BrowserThread::PostTask(
	creation_thread_,
	FROM_HERE,
	base::Bind(&ContentHashFetcherJob::DispatchCallback, this));
	}

	bool ContentHashFetcherJob::CreateHashes(const base::FilePath& hashes_file) {
	base::ElapsedTimer timer;
	if (IsCancelled())
	return false;
	// Make sure the directory exists.
	if (!base::CreateDirectoryAndGetError(hashes_file.DirName(), NULL))
	return false;

	if (!verified_contents_.get()) {
	base::FilePath verified_contents_path =
	file_util::GetVerifiedContentsPath(extension_path_);
	verified_contents_.reset(new VerifiedContents(key_.data, key_.size));
	if (!verified_contents_->InitFrom(verified_contents_path)) {
	verified_contents_.reset();
	return false;
	}
	}

	base::FileEnumerator enumerator(extension_path_,
	true, /* recursive */
	base::FileEnumerator::FILES);
	// First discover all the file paths and put them in a sorted set.
	SortedFilePathSet paths;
	for (;;) {
	if (IsCancelled())
	return false;

	base::FilePath full_path = enumerator.Next();
	if (full_path.empty())
	break;
	paths.insert(full_path);
	}

	// Now iterate over all the paths in sorted order and compute the block hashes
	// for each one.
	ComputedHashes::Writer writer;
	for (SortedFilePathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
	if (IsCancelled())
	return false;
	const base::FilePath& full_path = *i;
	base::FilePath relative_unix_path;
	extension_path_.AppendRelativePath(full_path, &relative_unix_path);
	relative_unix_path = relative_unix_path.NormalizePathSeparatorsTo('/');

	if (!verified_contents_->HasTreeHashRoot(relative_unix_path))
	continue;

	std::string contents;
	if (!base::ReadFileToString(full_path, &contents)) {
	LOG(ERROR) << "Could not read " << full_path.MaybeAsASCII();
	continue;
	}

	// Iterate through taking the hash of each block of size (block_size_) of
	// the file.
	std::vector<std::string> hashes;
	ComputedHashes::ComputeHashesForContent(contents, block_size_, &hashes);
	std::string root =
	ComputeTreeHashRoot(hashes, block_size_ / crypto::kSHA256Length);
	if (!verified_contents_->TreeHashRootEquals(relative_unix_path, root)) {
	VLOG(1) << "content mismatch for " << relative_unix_path.AsUTF8Unsafe();
	hash_mismatch_unix_paths_.insert(relative_unix_path);
	continue;
	}

	writer.AddHashes(relative_unix_path, block_size_, hashes);
	}
	bool result = writer.WriteToFile(hashes_file);
	UMA_HISTOGRAM_TIMES("ExtensionContentHashFetcher.CreateHashesTime",
	timer.Elapsed());
	return result;
	}

	void ContentHashFetcherJob::DispatchCallback() {
	{
	base::AutoLock autolock(cancelled_lock_);
	if (cancelled_)
	return;
	}
	callback_.Run(make_scoped_refptr(this));
	}

	// ----

	ContentHashFetcher::ContentHashFetcher(
	net::URLRequestContextGetter* context_getter,
	ContentVerifierDelegate* delegate,
	const FetchCallback& callback)
	: context_getter_(context_getter),
	delegate_(delegate),
	fetch_callback_(callback),
	weak_ptr_factory_(this) {}

	ContentHashFetcher::~ContentHashFetcher() {
	for (JobMap::iterator i = jobs_.begin(); i != jobs_.end(); ++i) {
	i->second->Cancel();
	}
	}

	void ContentHashFetcher::DoFetch(const Extension* extension, bool force) {
	DCHECK(extension);

	IdAndVersion key(extension->id(), extension->version()->GetString());
	JobMap::iterator found = jobs_.find(key);
	if (found != jobs_.end()) {
	if (!force \|\| found->second->force()) {
	// Just let the existing job keep running.
	return;
	} else {
	// Kill the existing non-force job, so we can start a new one below.
	found->second->Cancel();
	jobs_.erase(found);
	}
	}

	// TODO(asargent) - we should do something here to remember recent attempts
	// to fetch signatures by extension id, and use exponential backoff to avoid
	// hammering the server when we aren't successful in getting them.
	// crbug.com/373397

	DCHECK(extension->version());
	GURL url =
	delegate_->GetSignatureFetchUrl(extension->id(), *extension->version());
	ContentHashFetcherJob* job =
	new ContentHashFetcherJob(context_getter_, delegate_->GetPublicKey(),
	extension->id(), extension->path(), url, force,
	base::Bind(&ContentHashFetcher::JobFinished,
	weak_ptr_factory_.GetWeakPtr()));
	jobs_.insert(std::make_pair(key, job));
	job->Start();
	}

	void ContentHashFetcher::ExtensionLoaded(const Extension* extension) {
	CHECK(extension);
	DoFetch(extension, false);
	}

	void ContentHashFetcher::ExtensionUnloaded(const Extension* extension) {
	CHECK(extension);
	IdAndVersion key(extension->id(), extension->version()->GetString());
	JobMap::iterator found = jobs_.find(key);
	if (found != jobs_.end()) {
	found->second->Cancel();
	jobs_.erase(found);
	}
	}

	void ContentHashFetcher::JobFinished(scoped_refptr<ContentHashFetcherJob> job) {
	if (!job->IsCancelled()) {
	// Note: Run can result in ContentHashFetcher::ExtensionUnloaded.
	//
	// TODO(lazyboy): Add a unit test to cover the case where Run can result in
	// ContentHashFetcher::ExtensionUnloaded, once https://crbug.com/702300 is
	// fixed.
	fetch_callback_.Run(job->extension_id(), job->success(), job->force(),
	job->hash_mismatch_unix_paths());
	}

	for (JobMap::iterator i = jobs_.begin(); i != jobs_.end(); ++i) {
	if (i->second.get() == job.get()) {
	jobs_.erase(i);
	break;
	}
	}
	}

	} // namespace extensions